{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 实现混淆矩阵，精准率和召回率"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 准备数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn import datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "digits = datasets.load_digits()\n",
    "X = digits.data\n",
    "y = digits.target.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "将数据集改得极度偏斜"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "y[digits.target==9] = 1\n",
    "y[digits.target!=9] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=666)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 使用逻辑回归训练模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\CandyWall\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9755555555555555"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "log_reg = LogisticRegression()\n",
    "\n",
    "log_reg.fit(X_train, y_train)\n",
    "log_reg.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 计算混淆矩阵，精准率和召回率"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 自己实现"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_log_predict = log_reg.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "TN:True Negative,预测值为0（y_predict==0），并且预测对了（y_true==0）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "403"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def TN(y_true, y_predict):\n",
    "    assert len(y_true) == len(y_predict), \"样本真实值的个数和预测值的个数必须相等\"\n",
    "    \n",
    "    return np.sum((y_true == 0) & (y_predict == 0))\n",
    "\n",
    "TN(y_test, y_log_predict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "FP:False Positive,预测值为1（y_predict==1），但是预测错了（y_true==0）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def FP(y_true, y_predict):\n",
    "    assert len(y_true) == len(y_predict), \"样本真实值的个数和预测值的个数必须相等\"\n",
    "    \n",
    "    return np.sum((y_true == 0) & (y_predict == 1))\n",
    "\n",
    "FP(y_test, y_log_predict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "FN:False Negative,预测值为0（y_predict==0），但是预测错了（y_true==1）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def FN(y_true, y_predict):\n",
    "    assert len(y_true) == len(y_predict), \"样本真实值的个数和预测值的个数必须相等\"\n",
    "    \n",
    "    return np.sum((y_true == 1) & (y_predict == 0))\n",
    "\n",
    "FN(y_test, y_log_predict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "TP:True Position,预测值为1（y_predict==1），并且预测对了（y_true==1）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "36"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def TP(y_true, y_predict):\n",
    "    assert len(y_true) == len(y_predict), \"样本真实值的个数和预测值的个数必须相等\"\n",
    "    \n",
    "    return np.sum((y_true == 1) & (y_predict == 1))\n",
    "\n",
    "TP(y_test, y_log_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def confusion_matrix(y_true, y_predict):\n",
    "    return np.array([\n",
    "        [TN(y_true, y_predict), FP(y_true, y_predict)],\n",
    "        [FN(y_true, y_predict), TP(y_true, y_predict)]\n",
    "    ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[403,   2],\n",
       "       [  9,  36]])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test, y_log_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def precision_score(y_true, y_predict):\n",
    "    try:\n",
    "        return TP(y_true, y_predict) / (TP(y_true, y_predict) + FP(y_true, y_predict))\n",
    "    except:\n",
    "        return 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9473684210526315"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision_score(y_test, y_log_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def recall_score(y_true, y_predict):\n",
    "    try:\n",
    "        return TP(y_true, y_predict) / (FN(y_true, y_predict) + TP(y_true, y_predict))\n",
    "    except:\n",
    "        return 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall_score(y_test, y_log_predict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### sklearn中的实现"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[403,   2],\n",
       "       [  9,  36]], dtype=int64)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "confusion_matrix(y_test, y_log_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9473684210526315"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import precision_score\n",
    "\n",
    "precision_score(y_test, y_log_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import recall_score\n",
    "\n",
    "recall_score(y_test, y_log_predict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# F1 Score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 自己实现"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "F1 Score是精准率和召回率的调和平均值：\n",
    "$\\frac{1}{F1}=\\frac{1}{2}(\\frac{1}{precision}+\\frac{1}{recall})\\quad F1=\\frac{2\\cdot precision\\cdot recall}{precision + recall}$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def f1_score(precision, recall):\n",
    "    try:\n",
    "        return 2 * precision * recall / (precision + recall)\n",
    "    except:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision = 0.5\n",
    "recall = 0.5\n",
    "f1_score(precision, recall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.18000000000000002"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision = 0.1\n",
    "recall = 0.9\n",
    "f1_score(precision, recall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision = 0.0\n",
    "recall = 1.0\n",
    "f1_score(precision, recall)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 准备数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import datasets\n",
    "\n",
    "digits = datasets.load_digits()\n",
    "\n",
    "X = digits.data\n",
    "y = digits.target.copy()\n",
    "\n",
    "# 将数据集改成一个改成极度有偏的数据集\n",
    "y[digits.target == 9] = 1\n",
    "y[digits.target != 9] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=666)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 使用sklearn中的Fl Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9755555555555555"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "log_reg = LogisticRegression(multi_class=\"ovr\", solver=\"liblinear\")\n",
    "log_reg.fit(X_train, y_train)\n",
    "log_reg.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_predict = log_reg.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[403,   2],\n",
       "       [  9,  36]], dtype=int64)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "confusion_matrix(y_test, y_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9473684210526315"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import precision_score\n",
    "\n",
    "precision_score(y_test, y_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import recall_score\n",
    "\n",
    "recall_score(y_test, y_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8674698795180723"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import f1_score\n",
    "\n",
    "f1_score(y_test, y_predict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 精准率和召回率的平衡"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "决策边界：$\\theta^T\\cdot X_b=threshold$\n",
    "\n",
    "decision_function方法就是在求决策边界$\\theta^T\\cdot X_b$的值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-22.05700117, -33.02940957, -16.21334087, -80.3791447 ,\n",
       "       -48.25125396, -24.54005629, -44.39168773, -25.04292757,\n",
       "        -0.97829292, -19.7174399 ])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_reg.decision_function(X_test)[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "sklearn中的逻辑回归算法，$\\theta^T\\cdot X_b=threshold$，其中threshold默认值为0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_predict = log_reg.predict(X_test)\n",
    "y_predict[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "y_predict的前10个值和上面的decision_funciton函数预测的前10个值是一致的"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "decision_scores = log_reg.decision_function(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-85.68608522646575"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.min(decision_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19.8895858799022"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.max(decision_scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "可以自己指定threshold，然后将decision_scores中的值跟自己threshold进行比较，确定预测结果是0还是1。\n",
    "\n",
    "当指定threshold的值为5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_predict2 = np.array(decision_scores >= 5, dtype='int')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[404,   1],\n",
       "       [ 21,  24]], dtype=int64)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test, y_predict2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.96"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision_score(y_test, y_predict2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5333333333333333"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall_score(y_test, y_predict2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6857142857142858"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1_score(y_test, y_predict2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "当指定threshold的值为-5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_predict3 = np.array(decision_scores >= -5, dtype='int')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[390,  15],\n",
       "       [  5,  40]], dtype=int64)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test, y_predict3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7272727272727273"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision_score(y_test, y_predict3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8888888888888888"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall_score(y_test, y_predict3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7999999999999999"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1_score(y_test, y_predict3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 绘制精准率-阈值曲线，召回率-阈值曲线"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 解决绘图中文乱码问题\n",
    "plt.rcParams['font.sans-serif']=['SimHei'] #用来正常显示中文标签\n",
    "plt.rcParams['axes.unicode_minus']=False #用来正常显示负号"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "precisions = []\n",
    "recalls = []\n",
    "thresholds = np.arange(np.min(decision_scores), np.max(decision_scores), 0.1)\n",
    "for threshold in thresholds:\n",
    "    y_predict = np.array(decision_scores >=threshold, dtype='int')\n",
    "    precisions.append(precision_score(y_test, y_predict))\n",
    "    recalls.append(recall_score(y_test, y_predict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD6CAYAAAC1W2xyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xt4VNW5+PHvmktmck/IBYRAAAmIoAjGOyiKVEGttFrrOVSliNZqLT22VXvUU6vWS0WP/KpWqRQUrYp6pLVKwQsKCgKhFAW5awIJBnK/ZzKX9ftjT0IIgUySmey5vJ/nybN39qzZ864EXhZrr/1upbVGCCFE9LCYHYAQQojgksQuhBBRRhK7EEJEGUnsQggRZSSxCyFElJHELoQQUUYSuxBCRBlJ7EIIEWUksQshRJSxmfGhmZmZeujQoWZ8tBBCRKxNmzaVa62zumpnSmIfOnQoBQUFZny0EEJELKVUUSDtZCpGCCGijCR2IYSIMpLYhRAiykhiF0KIKCOJXQghoowkdiGEiDIBJXalVH+l1JrjvG5XSr2jlPpMKTU7eOEJIYTori7XsSul0oEXgcTjNLsd2KS1vl8p9Z5S6g2tdV2wgmxz8CvY9nbQTytExLPFwRlzID7d7EhMdaC6iaUF+/H5wveRn/lD+3H+yC7vMeqVQG5Q8gI/BP52nDaTgbv9+6uBfGBV+wZKqZuBmwGGDBnS3TgN5Tth9eM9e68QUcufxJIHwviZ5oZisqUF+3nqg90oZXYkx3bLBSean9i11rUA6vg/qUSgxL9fCfTv5DwLgAUA+fn5PfvndMz3jC8hxGHNNfDoEGiqMjsS0zW5vThsFnY+NM3sUEwVrIun9UC8fz8piOcVQnQlLhlQRoKPcS63D4dN0k+wfgKbgIn+/XFAYZDOK4ToisUCzlRoqgSvB3T4zi+Hmsvjw2G3mh2G6bqd2JVSFymlftbh8IvA75RS84GTgfXBCE4IEaCEfrDxBXgwA/50ntnRmMbl8cqInW5Ud9RaT/ZvPwI+6vBakVJqKsao/X+01t5gBimE6MJlT0JxARR9Cl9/bIzcraYUbw2J/ZWN1Da7ibNaGJGddMxrfi6PTMVAEMv2aq0PAEuDdT4hRDeceKHx9XmykdhdtcYoPgocqG5i0h+OWGRHQpyVuVPyOPfETE7JSW07bsyxy1RM9PyTLoQw5trBWCETJYn9YG0zAD+/aARZyQ4O1rpYu7ecR5bvAODs4f24+fzhjMtJo7HFg8MuI3ZJ7EJEk/g0Y/vpk5B01Krj3jtxCgzt2zn8xhZjZndiXhZnDjP+sfJ48/iypIZlm0t4Zf0+Pv/68IN7khyS1uQnIEQ0yRxpjNq3vBb8c/s8UPgZ3Lgi+Oc+jnqXB4BEx+EpFpvVwvgh6Ywfks4vLh7Jim2llNY289QHuzlrWHT8T6U3JLELEU0yToS794Xm3Euvh0M7gn7a0ppmZi3aQEOL54jjVqV4cMZYGv3HE+M6T1fpiXFce6ZxN/tNk4ZjtYTxbad9RBK7ECIwztSQ3AS1vbSWHaV1XDgqi/SEuLbjH+08xJ1vfsHYQcZ1gwRH1xdFE2UaBpDELoQIlDMNmqvB4zryuLL2amllXbMxIr/nstGMyE5uO/725mLuevNL3v/qIHE2CylOe48/I9ZIYhdCBCahH3ia4aHsI4/b4uGnnxnTQD1Q1+wGILlD4v7e+BymjT2BLfuryUp24JQ7SgMmiV0IEZjTZoKyGBdRW9UeMO54rfy6R4m9tKaZL4uN6Z1k59HpyGm3ctbwjB6HHKsksQshApOUDefNPfJY2S4jsfdw7v2Wlzfx7/3VAMTLiDxoZCW/EKLn2t8Q1QNVjS0APH71qV2VBhfdIIldCNFzrYl93TM9qioZb7dyyZj+/CB/cJADi22S2IUQPWd3AgqqvoHaki6bd1TT5JbVLiEgiV0I0TvXvGRsuzkdc6i2mW9rmkmJl8QebJLYhRC90zod080LqFv8q2FOzEoKdkQxTxK7EKJ3WguP1R+Eloajb2A6hpomY/36xBGZoYosZslyRyFE78T7i269OdvYWuzwk0+g/5jjvq3avyImVaZigk4SuxCid9IGw4znoOEQ1B+CdU9DxZ4uE3ttkxulOr8xSfSO/ESFEL132n8Y2+r9RmJvqu7yLdX+FTEWqcYYdDLHLoQIntb59gAupNY0uUlLkGmYUJARuxAieOKSjHoy2/9uXEztaOB4OOVqAKob3TK/HiKS2IUQwaMUDJ0EJZvg0PYjX/M0gzONooHT2HOonuKqRgamxZsTZ5STxC6ECK4b/t758RX3wKbFzF68kb1lDQDk58pj7EJBErsQom/YHOBp5pDLxXfHDWTOpGGM7J/c9ftEt8nFUyFEn1hbVA8+D40uFydmJXFqTpo8PCNEJLELIfrEqr11ADhwk54oF01DSRK7ECLkPF4fLoxk7qCFwf0STI4ouskcuxAi5Eprm3ERB8C7Pz2DgbnZXbxD9IaM2IUQIdXi8THxsVW4tDFiH5gkd5qGmozYhRAhVdFgVHscnN0PqoGPH4WEdssch10AJ003J7goJYldCBFSFfVGFcezzjoH1p4Au1ccfrGlEb7+WBJ7kEliF0KEVHm9MWJ3nnAy/HLHkS/+/XbYtdKEqKJbQHPsSqmFSql1Sql7j/F6ulLqPaVUgVLq+eCGKISIZJUNxog9I8lx9IvO1G4/eUl0rcvErpT6PmDVWp8DDFdK5XXS7DrgFa11PpCslMoPcpxCiAjVOhWTkRR39IvOVPA0QV0pNFYe+SUJv8cCmYqZDCz1768EJgK7O7SpAMYqpdKAwcD+YAUohIhcFfUuth2owW5VJDs6STcJ/sfiPTGq8xNc/RcYe1XoAoxSgST2RKDEv18JTOikzafAZcDPge3+dkdQSt0M3AwwZMiQnsQqhIgwP39tM5/tqWBoRgJKdbLMcexVgAav+8jjXjesvAeq9/VJnNEmkMReD7TW1kyi8+mb3wK3aK1rlVJ3AD8GFrRvoLVe0HosPz9f9zhiIUTE2F/ZxKS8TB676tTOGzhTIH/20ce9HiOxd0z4IiCBXDzdhDH9AjAOKOykTTpwilLKCpwFSOIWIsZprSmrczGqf3L3665brICSxN5DgST2ZcB1SqkngWuAbUqphzq0eQRjNF4D9ANeDWqUQoiI09DipcntJSu5k9UwXVEKrHbwtgQ/sBjQ5VSMf3plMjAV+IPWuhTY0qHNBuD4jyQXQsSU8jpj/XqPEjuANQ58niBGFDsCukFJa13F4ZUxQgjRpbL6XiZ2i01G7D0kRcCEECFR5h+xZ3Z2Y1IgrHEyx95DktiFECFR1uupGLsk9h6SxC6ECImyOhdWiyI9oZM7TgNhtYNPEntPSBEwIUTQ3f3WFywt2E9mkgOrpYf11y2yKqanJLELIYJu7d4K8rKT+a+pI3t+Eplj7zGZihFCBJXWmtLaZi4YlcWlYwf0/ERWmyT2HpLELoQIqpomNy0eH9k9vWjayiJz7D0liV0IETTVjS0sXlsIQP8UZ+9OZo83nrAkuk0SuxAiaOZ/uJunPjCqeg9I7WVid6aCqzYIUcUeSexCiKA55F+7DpDbL6F3J3OmycM2ekgSuxAiKPaW1fPuF99y0oBkNt5zMdm9nYqJT4PaEtBSLLa7JLELIYJi7Z5yAK6akNPzu03bS8gwth91LCYruiKJXQgRFLXNRiXG687JDc4JT59lbCu/Ds75YogkdiFEUNQ1e7BbFQ5bkNJKQj8YlA/N1cE5XwyRxC6ECIq6ZjfJTnvnzzbtKWeqXEDtASkpIIQIirpmD8nOIKcUZyoUF8DffmZ8b3PABXdBUnZwPyfKSGIXQgRFWZ2LjMQeVnI8luGTYf962PMhaC/UH4SBE2D8zOB+TpSRxC6ECIri6kYmDEkP7klPv8H4AmiqgseGypx7AGSOXQjRK16fpriqkW+rm8lJjw/dBzlSASVz7gGQxC6E6JUX1nzNxMdW4fFpBqX18m7T47FYwJkCNSVQfyh0nxMFJLELIXrl868r2vZzM0KY2AESs+DfL8O8PNj9fmg/K4JJYhdC9IrTbgXgj/8xnrOHZ4T2w65aCN/5vbFfUxzaz4pgktiFEL2yr7KRC0dlccW4gT1/DF6gBp4Gp/2nse9xHb9tDJPELoToMZ9Ps6+ikSG9reTYHTZ/cTFPc999ZoSRxC6E6LEbFm2gzuVhfLCXOR6PzV9gTBL7MUliF0L02PZv67BaVO+ebdpdFqvx2DxJ7MckiV0I0SNaa2qaWrhp0vC2C6h9xuaUOfbjkMQuhOg2rTWltc24vZr0BHvfB2B3yoj9OKSkgBCi2/70yV7+8M+dAGQkBeGhGt1lc4JbEvuxSGIXQnTbrtI6MhLjuOM7I5nWl/PrrWwO8DT1/edGCEnsQohuq2x0k9MvgZlnBelpSd3lSIHmWnM+OwLIHLsQotuqGlroZ8bceqv4NCkGdhwBJXal1EKl1Dql1L1dtHtWKXVFcEITQoSr0tpmMs2YW2/lTJXyvcfRZWJXSn0fsGqtzwGGK6XyjtFuEjBAa/1OkGMUQoSJgsJK5r62mbI6FyOyk8wLxJlmVHlceS/4fObFEaYCGbFPBpb691cCEzs2UErZgT8DhUqpKzs7iVLqZqVUgVKqoKysrIfhCiHM9NcN+3j3i2/Jy07i/JFZ5gVy4oVGCd+1f4SafebFEaYCSeyJQIl/vxLo30mb64GvgD8AZyqlbu/YQGu9QGudr7XOz8oy8Q+EEKJH3v/qIGt2lzNmYArv33EBo09IMS+Yk6+Ey5409mWu/SiBrIqpB1ofi5JE5/8YjAcWaK1LlVIvA78H/tidQNxuN8XFxTQ3y9rUUHI6neTk5GC3m3jhS0ScyoYWbnqpAIBTBqWaHI2f0x9Hk8y1dxRIYt+EMf3yOTAO2NlJmz3AcP9+PlDU3UCKi4tJTk5m6NChKBXi0p8xSmtNRUUFxcXFDBs2zOxwRATZ8e3hpYUhffxdd8SnGVsZsR8lkMS+DFijlBoITAOuVUo9pLVuv0JmIfAXpdS1gB24uruBNDc3S1IPMaUUGRkZyDUO0V3FVcbNQE//53imnNTZbKwJWkfssjrmKF0mdq11rVJqMjAV+IPWuhTY0qFNHfCD3gYjST305GcseqKs3ii4dfHo/n1f8OtYnDJiP5aA1rFrrau01kv9SV20s27dOurr63n33XeDet6CggLWr1+P1hqPx3Pctu2vS7jdbtxud1BjEeKv6/eRGGcNn6QO4EgGZZE59k7InaftaK3RWjN16lRcLhcHDhxg165dXHnllaxbt46dO3eydetWFixYAIDP52PWrFm4XC5WrVrFsmXLjjhf+wQ7f/58Xnvttbbvu0rWFouFG2+8kS+++ILLLruMyy+/nOnTpzN9+nRGjhzJ5s2b29rOmDGDTz75hMLCQhYtWsTs2bMpLCxk7969XX6OEF1pdns5UNNEbkai2aEcSSn/jUoyYu9IasW08+qrr9LQ0IDD4WD69Onceeed7Nu3j/LyctavX4/T6eS6667jvvvuIy0tDbvdTnl5OTNnzsTlcvHFF1/wxBNP8PTTTzNu3DjOPfdc4uPjsVgs7Nu3jyFDhvDcc8+htaaxsZHVq1cTH9/5hagJEyawcOFCxo0bx4oVK7jggguYMWMGo0aNYsOGDdhsxq9u7969OBwOXC4Xb7zxBhs3bsTlcvHmm2/i8Xi47bbbSE5O7ssfo4gyO0vr0Bpuv2iE2aEczZkKG/8MU+47POcuJLG3d8011/Dcc89RV1fHVVddhcPh4M0332Tfvn28++67nHDCCdxyyy0sWbKETZs28bOf/YwrrriCxYsXc+211/Loo48ydOjQtvNdeumlnHnmmZSXl7N3716GDx/O4MGDKSkpYe/evcdM6iUlJTzwwANUV1dz8OBBAL788kuamppISkri/PPPb5sr/+///m9Gjx7NxRdfzCOPPEJxcTEWi4Wamhruu+8+Seqi155ZtQeAseGyzLG97JOhqhD2b4C8qWZHEzbCMrH/7p1tfHUguJXbTh6Ywm+vGHPcNp988gnr16+nsLCQxsZGCgoKmDx5Mna7nTPOOAOlFO+99x7Tp0/HbreTl5fH+++/z6WXXsqWLVu4/vrrsdvtfPjhh4CRdAsLC3nwwQdpaGhgxIgRvP7667z33nvcf//9x4yjdQR+6623UlZWRlZWFvPmzeOuu+4C4IMPPgDgjTfeYMuWLQwbNgyLxUJDQwNLliwB4N1336WqqioIPzkRy5rdXlZ+dZD0BHv4LHNs7+Lfwc73ZDqmA5ljb2fKlCnk5OSQnZ1NfHw8Z599NitWrKC+vp5Vq1ZRWlrK9OnT2b59O0VFRTz88MNMnTqVf/7zn1xwwQW89NJLbUkdYMeOHbz44os89thjjB8/nl/96lfcdNNNrFq1ihdffBGAn/zkJ0yePLnt64EHHmh7/9NPP82AAQP43//9X1JSUpg/fz5JSYfrc4wZM4annnqq7fumpib27NnDnj17OHToUB/8xES0W7HNWC9x24UjwnNFlSx57FRYjti7GlmHSmVlJRs2bCA7O5vVq1fj9XoZO3YskydPBuDPf/4zAIsWLeLiiy8G4KOPPmLGjBkUFBQwZ84c7HY7y5cvp6amhueff57f//73zJkzh5qaGubNm4fdbuehhx7i1VdfpaWlheeff/6oOMrLywHjAuu9995LZmYmF110EU8++SSnnXZa20Xak08+mcbGxrb3ffvtt7zwwgsAlJaWMnWq/NdU9E7r+nXT6q53pTWx15Qcv12MkRF7O0uWLOG3v/0tSileeeUVRo8eTXp6Ojk5OeTk5GC1WqmsrOT9999vS+wXXXQRy5YtY+LEibzwwgssX74cgLfeeotvvvmGG2+8Ea01ZWVlHDhwAI/Hw9y5c1mzZg0vvfTSMWNxu9089NBD3HrrrUycOJHy8nIcDgeLFi06ZtmFzMxMZsyYwYwZM5gwYULwf0AipjS2eHh8xU6SHTbi48JomWN7dqex/dex/y7ForAcsZtl7ty5ADz22GM4nU48Hg9vvPEGa9asAcBqtbJ69WpmzpyJxWJBa83y5cuZOHEiX3/9NT/4wQ+wWCxcf/313HbbbcyePbvt3E899RQDBgzg2muv7TIOj8dDQ0MDTz75JFOmTGHIkCEsWLCAxx9/nCVLljB48OC2/xZrrfH5fHi9XlJTU5k40Si+WV1t/NfU6/VitYbpX0oR1jbvM/4MnTo4DC+atpdzJlRLhcf2JLF3onV6w+12c/fddzNr1iyeeuopdu3axYwZM9raeb1epk2bxuLFi7s8p8vlCnhNeUNDA2lpaSxcuBAw1stPmTKF1NRUVq5cyUcffdR2IdXlclFeXs706dPJyso64qLsxo0b8Xg8Af1jIkRHuw7WAfDw904xOZIuDD4TDm41O4qworTWff6h+fn5uqCg4Ihj27dvZ/To0X0eS2daWlqIi4s74li4jHyDEUc4/axFePpox0FmLzb+ju548NLwuuO0o08eh1UPwb1lYIvrun0EU0pt0lrnd9VO5tg70TGpA2GR1CF84hDRbdUOo1Dcwhvywzupw+Eqj0uvhw1/NjeWMCGJXQhxlI2FlUzKy2TK6DCp5Hg8Q86GAafCvrXw6VNdt48BktiFEG3qXR6WrCtk58E6zhjaz+xwAjPgFLhlDZw2U25U8pPELoRo8+r6fdz3t20AXGDmM017wpkGLXXglcJ3ktg78Hq9ZocASNleYY73tx9kWGYiX95/CeMGp5kdTve03YUqo3ZZ7tjBHXfcwdatW2lqamL37t2MGzcOMO7kvPrqq7n//vsZPnw4Q4YM6fT9LpeLdevWAUaSbX226Pz58+nfv3/b0kOPx9NWobEzFouFWbNm8corr3DnnXdit9vx+XwA7Nmzh9dff53x48cDRtne3/zmN+Tm5rJy5UrWrFnDgw8+iNfrJTc397ifI0SrLfur2fBNJdeeMZgkRwT+mUnwTx1tfRPO+om5sZgsAn97oTV//nwAli1bxpo1a3jiiScAeOaZZ9pWpAwZMoSPP/640/dfeOGFbftStldEkg+3G5VEb7ngRJMj6aGTLje28uANSeyd0Vrz7LPP8vDDD7cd279/P5MmTQJoS6qzZs2iuLgYMApyzZ8//4jliFK2V0SST3aXM35IGkMzw+yBGoGKS4C4JJmKIVwT+/K7ofTL4J5zwCkw7dEum1VUVDBnzhzy8vLIzzfuA/j1r3/Nyy+/zC9/+csj2paXl7eV0G0tFNaelO0VkaKqoYUviquZOyXP7FB6x5kmlR6Ri6dHWLt2Leeddx6XX345Tz/9dNvxGTNmsHbtWrKyurdKQMr2ikixZk85WkfgSpiOnKmw5bWu20W58ByxBzCyDoVzzz2XrVu3MmDAABYtWnTEayUlJTzyyCNH1F3xer1tI/X+/Y+8kUPK9opI8snOMtIS7JyaE2ErYTqKSwDthZZGYz9GhWdiN5HNZiMnJ4dPP/30iOPz5s1rm1tvXZ3SWqK3vdbaO+3L9gKUlZVRV1fHoEGDmDt3Lm63m8zMTObMmdNpHK1le7dt28aBAwe6VbYXYMOGDT3ovYhFWmtW7y5jUl4WVksYPkyjO8b9BxRvNObZJbGL9srKyo6aM9+/fz+PPfYYAMXFxW312DtqTbqzZ8+Wsr0iIuworaOszhX50zBwuG5Mcw2knGBuLCaSxN6JzMzMo5Yzzps3r+3Gn9dee+2YD7LoWLWylZTtFeFqm//5whOGRPg0DBy+SalkE3hdoKyQPRossTW4kbK9EUbK9opge+yfO3hhzddsf+BSbNYIX09RuhWeO+/IY9PnwZk3mRNPkAVatjesRuxa6/B8YG4Y6W1SN+MfchHe/lVURW5GYuQndYD+Y+CGf4DL+F8IS2+AmmJzYzJB2CR2p9NJRUUFGRkZktxDRGtNRUUFTqfT7FBEmCiqaGD9N5VcMiYCyvMGQikYNunw9/Gxua49bBJ7Tk4OxcXFlJWVmR1KVHM6neTk5JgdhggT//jiWwB+OnmEyZGEiDMtJu9EDZvEbrfbGTZsmNlhCBEztpbU8PiKnZyem85pkVbJMVDOVNj2Nly9yBjNx4gomFQTQvTEI8u3A3D9ObkmRxJCzhRj21hpbhx9TBK7EDFma0kNZ/7+Az7bU8HPp+Rx5WmDzA4pdE65xtjG2Dx7QIldKbVQKbVOKXVvF+36K6U2Byc0IUQoLFlXRL3Lw02ThkX3aB3a3bAkif0ISqnvA1at9TnAcKXU8cq/zQM6r0UrhDCV1ppnVu3h9YL9TD/lBO657GQykxxmhxVaMfpUpUBG7JOBpf79lcDEzhoppS4CGoDSoEQmhAiqkuomHl+xE4Drzo7ykXorp3/EHmMP3wgksScCJf79SuCoBa9KqTjgPuDuY51EKXWzUqpAKVUgSxqF6HtPrNwFwLLbzou855n2lIzYj6mew9MrScd4z93As1rrY/6zqLVeoLXO11rnd7euuRCidxpcHt7eXEJGYhzjclLNDqfvtC8KFkMCSeybODz9Mg4o7KTNxcBtSqmPgdOUUi8EJTohRK9t3lfFmN+uAGDuxXmxdWe3PQEstpi7eBrIDUrLgDVKqYHANOBapdRDWuu2FTJa6/Nb95VSH2utOy8yLoTocxu+MdZw//qSUVx/zlBzg+lrShnz7F++CWW74NJHID36ry90OWLXWtdiXED9HLhQa72lfVLvpP3koEUnhOi1XQfr6ZcYx20XRmnZgK6M/5ExJbPzXfhmtdnR9ImA1rFrrau01ku11rLiRYgI4vVpVm4rZVJeptmhmGfq72DWe8Z+jEzJyJ2nQkSxRZ99Q53Lw+RRMb5gIS4JlCVmLqJKYhciSrV4fDyyfAcA08bG7mPiALBYwJEM+9ebHUmfkMQuRBSqd3mYvXgjXp/mxonDcNpj69FwnbI5jTn2g9vMjiTkJLELEYVe/ryIT/eUMywzkZ9POV4VkBhyycPGtvZbc+PoA2FTj10IERxr95bz6PIdnHxCCu/NndT1G2LFgFONbQxcQJURuxBRxOP18Zv/+xKAOy8dZXI0YaatvIAkdiFEBLnrrS8pqmjk5vOHM3lUttnhhJfWxL7uWdi2zNxYQkwSuxBRoqLexVv/KmbyqCzumDrS7HDCj90JE66HulL44nWzowkpmWMXIgporfnx4o0AzJ2SJ6tgjuW7f4TKb6K+jK+M2IWIAl99W8sXxTX84uI8xg9JNzuc8OZMjfoblSSxCxHhtNbMfGE9CXHW2Cvy1RPOtKi/gCqJXYgI9+/91VQ3ujljaD/6JcaZHU74i0+TEbsQIry9s8W44ebxq081OZII4UyFlnrwesyOJGQksQsRwYoqGlhasJ/vjhtIdorT7HAiQww8Lk8SuxARqqbRzYxnPqPF64vdWus90fqA6yieZ5fELkSEuv21zVQ1unnuRxMYNSDZ7HAiR+uI/a0b4cUr4OuPTQ0nFCSxCxGByupcbCqsZOygFC46qb/Z4USWQadD3neM56EWrYPt75gdUdDJDUpCRJhPd5fzi9f/jVdrHr96nNnhRJ6kLJj5hrE//7SonGuXxC5EBNlUVMXsxRuxWOC5H53O6BNSzA4pskXpzUqS2IWIAFprlhbsZ/4Hu8lKdvCP2yeSLmvWe8+ZCjXFULwJBk0ApcyOKChkjl2ICLBq5yHueutLDtQ0c9/loyWpB0vKIDj0FbxwERStNTuaoJERuxAR4KV1RfRPcfDxry4kPk4KfAXNtEdh+AXw9k+gLnqerCQjdiHCnNaarSU1TByRJUk92JypMHyysR9Fc+2S2IUIc8VVTZTXtzB2kFwoDYkovGFJErsQYay83sXsxRtx2CxMyssyO5zoZHeC1QEb/wJ/vx20NjuiXpPELkSYKixvYM6LBRRVNLL4x2cyIjvJ7JCi19m3gC0O/vUSuBvNjqbX5OKpEGFGa80zq/Ywb+UuAH59ySjOOTHD5Kii3NQHIH0Y/OMXxlx7XKLZEfWKJHYhwsgbBfv564Z9bN5Xzem56dx72Wh5IlJfaa0h01QNKQPNjaWXJLELESb+6/V/8/bmEqwWxT3TRzNn0jC3FKtQAAAMmklEQVRUlNwwExHi/RdRv1kN/U82N5ZeksQuRBj4ZFcZb28u4bvjBvLENeOwW+XyV59LGWRs/3kXjLoU0oeaGk5vyJ8eIUxUVNHAzS8VcMNfNpDssHH3tJMkqZslaxRMfdDYrztobiy9FNCIXSm1EDgZeFdr/VAnr6cCrwFWoAH4oda6JZiBChFtlnxexP1/34ZVKaaclM3jPxgnzyw1W+55xjbC17R3mdiVUt8HrFrrc5RSf1FK5Wmtd3doNhN4Umv9vlLqT8ClwN9DEK8QEa+m0c0dS//NhzsOMX5IGv/v2vEM7pdgdlgCouaxeYGM2CcDS/37K4GJwBGJXWv9bLtvs4BDwQhOiGizZX81t77yL0qqmzg9N50lN55JQpxc6gobrRdQ/+8mGHwWpOeaG08PBTKZlwiU+PcrgWM+rkUpdQ6QrrX+vJPXblZKFSilCsrKynoUrBCR7OXPi7jm+XW4vT6ev+503vrpuZLUw01CxuHpmAP/MjeWXggksdcD8f79pGO9RynVD/gjMLuz17XWC7TW+Vrr/KwsuTVaxJb5H+zm3mVbOXt4Bu/+fBKXjBlgdkiiM0rBlU8b+x6XubH0QiDDhU0Y0y+fA+OAnR0bKKXigDeA32iti4IaoRARRmtNZUML5fUt7D5Ux0tri9hQWMnlp57AUz88DZuseglvNqexdTeZG0cvBJLYlwFrlFIDgWnAtUqph7TW97ZrcyMwAbhHKXUP8Cet9evBD1eI8FVS3cRf1xfx/lcH2XWwvu14Tno8s84dyt3TTpKkHglaE3s0j9i11rVKqcnAVOAPWutSYEuHNn8C/hSSCIUIUx6vj+dXf81ne8qpqG9h58E6APKyk/jVd0aSm5HYVpVR6qhHEJvD2HqazY2jFwK6cqO1ruLwyhghYlpji4dFnxXy3Cd7qWv2MGZgCv0S47hj6kjOG5HJ6blS2yWixcKIXQgBDS4Pe8vq2Xagloff205ds4czh/XjhnOGctmpJ5gdnggmixUsdvBE9xy7EDHH59MUFFXxzpYDfLqnnG/KG9peG9k/iQXXjZVSutHM5pQRuxDRQGvNhm8qWfjpN6zbW0Gdy4PTbuHcEzP5/vhB5GYmMjwzkZH9k4mzyUXQqGa1w5bX4NJHzI6kRySxCwEs21zCb/++jZomN1nJDi4fN5BzTsxgyknZJDrkr0nMiUuK6Hox8idWxKSaRjc7Smv5sqSGtzeXsO1ALafnpnPVhBxmjB8od4TGunE/hNXzwOcDS+T970z+9Iqo5/H6OFTn4p0tB1ixrZSqRjeFFQ1tzyweNziN/7n8ZH50dq5MsQiDMxXQ0FJ3uDBYBJHELqLCN+UNfFvdxNYDNew91MCOg3VUNrhocHmpbDhcQfrUnFROPiGFGacNYvyQNHLS4xmeJQ+JFh04/cXAtr8DCZmHj1vtMOx8YxvGJLGLsOX1aWqb3OyvasTl8VHV0MLWA7UUVTTg9vpo8WgqGlzsKq2jocXb9r6sZAcD0+LJz+1HfJyV7GQHafF2JuZlMSJbkrgIQGqOsf3bbUe/dtVCOOXqvo2nmySxiz7V4PKwv6qRnaV1HKhupq7ZTUV9C2X1LqobW2jx+qiob+FgbTM+3fk5BqY6SXTYsFstpMTb+EH+YIZlJpKXnURe/2Sykh192ykRfYZPhls/P7JejLsRFl8GdaVmRRUwSewiKFweL7VNHqoaW6h3eSiqaKC0xkVxVSOFFQ0UVTTS4PJQ1eg+4n1Wi6JfYhxZSQ7SE+2kWu3k9kskNyMBh82Kw25haEYiSQ4bSU4bedlJskpFhJ5SkD36yGM+H6Ai4iEc8jckhnm8PsrrWyitbcbj9eHxaVweH7VNbuqaPZTVufD6jONen6bO5aHZ7cXr03h8mvpmDw0uDwfrmtlf2fldemkJdnL7JZCfm06S00ZGooMTs5MYmpHAiVlJJMRZUUr1cc+F6AGLBZwpEbEMUhJ7GPH6NE1uL/XNHupdbupdxn5ji8c47vLQ1OLF5fHh8vhodntpcHlwe314vJoW/9bl8dLQ4qXF4zO+vMa2ra0/Mbd4fF3GZFFgs1iwWhQJcVacdis2q8JmUSTE2UiJtzEuJ42rJuSQkRhHWkIcSQ4bOenxDEyLl9G1iC7OVBmxRxOvf4Tq1Rqf1vh8Gp8GnzZGs1rT9prb66OszkWDy0O9y0tTi6ctGdc1e/i2pomqRjfNLV4aWoyRcXWTO6BE257TbiHJP9dssyrsVgt2i4U4m4VEh5WUeDtxVkWczYLdaiHRYSPOasFmUVitini7lRSnnaGZCdgsxvE4m4XUeDuJDhvZyQ4pMytEe840aJIRe1BtLanhzU3FbdMBXn+C9WqjtkfrMY/XR7Pb1+51fxL2t/G1JWCNy+3F50/K2t+urU275N3i9bWte+4Nu1UxINVJv0QHCXYr/VOcjBmYQnpiHAl2G/FxRgJOaveV6LARH2clMc5GgsOKw2YhzmqRKQwh+pqM2IOvpLqJtzYVEx9nxWZRWCwKi1JYLQqLwr81Rq5OuzF9YLNZ2o63b2NRCrvNgtP/uvK/3no+1WHfYbOS4rQZo9229gqrBZRSWJXCYjn8nqwkB8lOO4kOKwlxNpx2Cw6bFbtVSUIWIlLFp0H5brOj6FJEJfZLxgzgkt/JsyKFECZxpkLZDmiuNS6khimZQBVCiEClDja26542N44uSGIXQohATfqVsa0/aG4cXZDELoQQgbLaICMv7C+gSmIXQojuiA//JY+S2IUQojucqfD1Kvh2i9mRHJMkdiGE6I4x3ze2RWvNjeM4JLELIUR3nPpDYxvG8+yS2IUQojusNohLDut59oi6QUkIIcJCfBoc3ArblgX+nqT+kHtO6GJqRxK7EEJ0V+pgKFxjfAVMwZ1fQ0K/kIXVShK7EEJ018ylUL0/8PZ7PoD374OGcknsQggRlhzJ0P/kwNvXHjC2ffSQDrl4KoQQoeZMNbZ9dMFVErsQQoRafJqxfWcurA19ATGZihFCiFBLHwZnzIGGMkjKDvnHSWIXQohQs9rgsif67OMCmopRSi1USq1TSt3bmzZCCCFCr8vErpT6PmDVWp8DDFdK5fWkjRBCiL4RyIh9MrDUv78SmNiTNkqpm5VSBUqpgrKysu5HKoQQIiCBJPZEoMS/Xwn070kbrfUCrXW+1jo/KyurJ7EKIYQIQCCJvR6I9+8nHeM9gbQRQgjRBwJJwJs4PLUyDijsYRshhBB9IJDljsuANUqpgcA04Fql1ENa63uP0+bs4IcqhBAiEF2O2LXWtRgXRz8HLtRab+mQ1DtrE74V6IUQIsoprXXff6hSZUBRn3/wYZlAuYmf31dioZ+x0EeQfkaT3vQxV2vd5eoTUxK72ZRSBVrrfLPjCLVY6Gcs9BGkn9GkL/ooq1eEECLKSGIXQogoE6uJfYHZAfSRWOhnLPQRpJ/RJOR9jMk5diGEiGaxOmIXQoioJYldCCGiTMwkdqVUulLqPX+FyefbHY+6OvJKqWeVUle0+z7q+giglOqvlNrc7vuo6qdSKlUptVwptVIp9bZSKs5/PKr6CdHZp1ad/R5D3d+YSezAdcAr/vWjyUqp/GisI6+UmgQM0Fq/4/8+6vrYzjz8xeeitJ8zgSe11t8BSoFLo7Gf0dinDjr+Hq8lxP2NpcReAYxVSqUBg4H9BFZrPmIopezAn4FCpdSV/sOTiaI+tlJKXQQ0YPxFgSjsp9b6Wa31+/5vs4BDRGE/ic4+tenk9/gjQtzfqH3mqX+6ZVS7Q6uAXODnwHaMuvEd68hP6MsYe+sYffwK+ANwu1JqCBHeR+i0nx8BFwLfwyhAB1HaT631A0qpc4B0rfXnSqmbiPB+diLif3eBaP09YlS/DWl/ozaxa61/0v57pdRfgFu01rVKqTuAHxPhdeQ76ePTwAKtdalS6mXg9xj/M4nYPkKn/fwf4FmtdbVSqvVwRP8u4eh+Aiil+gF/BK7yH4r4fnYiGvt0hA6/xzsIcX+j7gd4HOnAKUopK3AWoIm+OvJ7gOH+/XyMQmvR1keAi4HblFIfA6cppV4gCvvpv1j6BvAbrXVr0byo6yfR2ac2nfweQ97fmLlBSSl1JrAIYzpmHcZ/4y3AGuBD/HXkI7nksFIqGfgLxqMJ7cDVQB1R1MeOlFIfa60nK6VSiLJ+KqV+CjwMbPEf+hOwnOjrZ9T97trr5Pe4CGPUHrL+xkxiPxalVDowFVittS7tqn0kioU+gvQzkkVjn44n1P2N+cQuhBDRJpbm2IUQIiZIYhdCiCgjiV0IIaKMJHYhhIgyktiFECLK/H+yQhtYvU9yOAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(thresholds, precisions, label=\"精准率-阈值\")\n",
    "plt.plot(thresholds, recalls, label=\"召回率-阈值\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 绘制精准率-召回率曲线"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD6CAYAAAC1W2xyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAEvNJREFUeJzt3XuMX2Wdx/H3t9ML7bSlt6Gl0FKucVEo6CwutJiBFSNh1V0iF9dgFHeJrmKyJpug4rqXJiaskuyyYqyyG7MaI7iCEmCtCSAXwd3pYkUEobKt0KF02unQdjrToTPP/jG/OrTO5fx+/fV3eeb9SiY97Xl+53zztP3MM8855zmRUkKSlI9p9S5AklRdBrskZcZgl6TMGOySlBmDXZIyY7BLUmYMdknKjMEuSZkx2CUpM9PrcdIlS5akVatW1ePUktS0Nm7cuDOl1DZZu7oE+6pVq+js7KzHqSWpaUXE1iLtnIqRpMwY7JKUGYNdkjJjsEtSZgx2ScqMwS5JmSkU7BGxNCIenWD/jIi4NyIej4jrq1eeJKlck97HHhELgW8CrRM0uxHYmFL6u4i4PyLuSintrVaRh/x6+17u+0VXtQ+blZWLW3n/206udxmS6qjIA0pDwDXADyZo0wHcVNp+BGgHHnpjg4i4AbgBYOXKleXWCcDmHfu47aHNFX12Kjj0+to/PW8501ucZZOmqkmDPaW0ByAiJmrWCmwrbfcAS8c4znpgPUB7e3tFb9C+4twTueLcKyr56JTw5Q2/5isPbaZl2oR/V5IyV61h3T5gdml7bhWPqzIMvD7ErOktk30TlpS5agXwRmBtaXs1sKVKx1UZDhwcZtaMaRwcGh73K6WKfliS1ETKXgQsIi4Fzk4p/esb/vibwP0RcTFwNvCzKtWnMhwcTvTuf50zPvfAuG1Wr1jADz6xpoZVSaq1wsGeUuoo/fog8OAR+7ZGxGWMjNr/NqU0VM0iVcxH157KifOPG3f/Q7/ewQuv7qthRZLqoWrL9qaUuoA7q3U8le/0trnc+Mdnjrt/9/7Xed5gl7LnRc4pJALn2KUpoC4v2lB9BDA4NMw//ei5epeSlWkRXPW2FaxcPKfepUiAwT6lnLVsHkHwtZ+8WO9SsjGcEsMJFs6ZyfVrT613ORJgsE8pV7ev4Or2FfUuIyt/f+8zfOvJrfzJuSfWuxTpd5xjlyq078BB7up8mSvOOZETJrgbSao1R+xShf5z48vsO3CQD1ywkgMHq3eH74xp05jmshA6Cga7VKF7N42sNHrN+ieretzzVy7g+x+/yKUhVDGDXarQp991Fk/9treqx3z+1b384OddPPvKXs5ePr+qx9bUYbBLFbro9CVcdPqSqh6zp2+Q+37xCj/c1GWwq2JePJUayKLWmVx85hLu3dTF8LAPk6kyjtilBvPe85bz19/dxOfueZr5s2eM2eZ9q09yRK9xGexSg7ns7GWcsvgFvv+/235v33BKvD40MpI32DUeg11qMHNnTecnf3PJmPu+8eiLrLvvWd67enmNq1IzcY5dahLDw4lv/+y3vHXlAt68/Ph6l6MG5ohdahI//c0u/m9nH3/5Z+ewf/BgVY7ZMi2YNb2lKsdS4zDYpSbxwC9fAeCzdz/NZ+9+uirHnDV9Gvd9ai1nnDCvKsdTYzDYpSbxkTWrWLmoeksD/3BTF8+/updFrbOqdkw1BoNdahJnnDCvaiPrwYPDrH/kRS47eymLWmdW5ZhqHF48laagB5/bwa6+Qa56m8s458gRuzQFfW/jSwD85PluHt+8s87VTOzkhbP58BpfYlIOg12agmbNaKF1Zgt3db5U71ImNHBwmJYIrrtwFS0uZVyYwS5NQV/587fWu4RC3nPbY8ye0WKol8k5dkkNqXf/IL/seo2Lzlhc71KajsEuqSE9+eIuUoI1Z1R3aeSpwGCX1JB++ptdzJnZwuqTF9S7lKZjsEtqSI9v3skfrlrEzOnGVLnsMUkNZ/trA/ymu481zq9XxGCX1HCeeHHk3vpqv3pwqvB2R0kN5/HNuwC4q/Mlvrfx5Zqcc+GcmXzy0jOyuLXSYJfUcJbMncX846Zz91O//xapY+HAwWEOHBzmqvaTWb5gdk3OeSwZ7JIazk2Xv4mbLn9Tzc73xQee5d8f28LS+cfV7JzHUqE59oi4IyKeiIibx9m/MCLuj4jOiPhadUuUpGNry84+ViyancU0DBQI9oi4EmhJKV0InBYRZ47R7Drg2ymldmBeRLRXuU5JOma27trP8gWz2d03OO7X8HCqd5mFFZmK6QDuLG1vANYCLxzRZhfwlohYAKwAGntlIUl6g+17Bnhu+17O/8cfj9vmAxes4ItXnlvDqipXJNhbgUNXMHqAsVYPegy4AvgU8Gyp3WEi4gbgBoCVK1dWUqskHRP/cu35vNi9b9z9X97wPP2DQzWs6OgUCfZ9wKHLxHMZe/rmC8DHUkp7IuLTwEeA9W9skFJaf+jP2tvbm+dnGknZe8dZbbzjrLYx9w0NJ9bd92xT3S1T5OLpRkamXwBWA1vGaLMQOCciWoC3Awa3pCzs3HeAg8Mpu2C/B7guIm4FrgaeiYh1R7T5IiOj8deARcB3qlqlJNXJtt5+AE5qomCfdCqmNL3SAVwG3JJS2g5sOqLNfwNvPiYVSlIddZWCvZlG7IUeUEop7Wb0zhhJmjJGg715Hl5yETBJmkBX7wDzZk1n3nEz6l1KYQa7JE1gW29/U03DgMEuSRN65bX+ppqGAYNdkibU1TvgiF2SctE/OERP36DBLkm56Hqt+e6IAYNdksb1u1sdj3fELklZaMaHk8Bgl6RxdfUOEAHLjncqRpKy0NXbz9J5xzGjpbmisrmqlaQa6nqtnxOb7MIpGOySNK5tu5vvqVMw2CVpTEPDiW29/axYOKfepZTNYJekMWzfM8DrQ4lTFhvskpSFrbv6AFi5qPmCvdB67JI01bzUsx+A/3hiKz/8edeEbf/gxHl8eM2ptSirEINdksawYtEcTlowm5+/1Dthu97+Qf7rmRaDXZIa3UWnL+Hxmy6dtN2N33mKp1+eOPxrzTl2SToKPX0HWNQ6s95lHMZgl6SjsGvfIItaZ9W7jMMY7JJ0FHr6BlnU2ljvQzXYJalCKSV273fELknZ2HvgIK8PJRY7xy5JeejZNwjgxVNJykXPfoNdkrLiiF2SMtPTZ7BLUlYOTcUsnmuwS1IWevoGOW7GNObMbKzVWQx2SarQrn2DLJrTWKN1MNglqWI79g7QNr/x3olqsEtShXbsOcDSeY311CkUDPaIuCMinoiImydpd3tEvKc6pUlSY9uxd4AT5jdhsEfElUBLSulC4LSIOHOcdhcDy1JK91a5RklqOAcODrF7/+ssndecUzEdwJ2l7Q3A2iMbRMQM4OvAloh431gHiYgbIqIzIjq7u7srLFeSGkP33gMAzTliB1qBbaXtHmDpGG0+BPwKuAW4ICJuPLJBSml9Sqk9pdTe1tZWab2S1BBe3XMo2JtzxL4PmF3anjvOZ84H1qeUtgPfAi6pTnmS1Ji69w4ANO1UzEZGp19WA1vGaLMZOK203Q5sPerKJKmBjY7YG28qpsjjUvcAj0bEcuBy4NqIWJdSeuMdMncA/xYR1wIzgPdXv1RJahw79g4wfVo05ANKkwZ7SmlPRHQAlwG3lKZbNh3RZi9w1TGpUJIa0Kt7DtA2bxbTpkW9S/k9hRY4SCntZvTOGEma8l7dM9CQF07BJ08lqSLdew9wQgM+dQoGuyRV5NU9AyxtwAunYLBLUtkGXh956nSZUzGSlIdtvf0AnLRw9iQt68Ngl6QybdtdCvYFc+pcydgMdkkqkyN2ScrMtt39tEyLhlyLHQx2SSpbV28/y+Yfx/SWxozQxqxKkhrYy739nLSgMadhwGCXpLJt293fsPPrYLBLUlkODg2zfc+AI3ZJysUrrw0wNJw42RG7JOVhy64+AFYtaa1zJeMz2CWpDFt2jgT7qQa7JOVhy679zJ7R0rArO4LBLkll2bKzj1MWzyGi8V6wcYjBLkll2LKrj1WLG3caBgx2SSpsaDjxUk9/Q184BYNdkgrr6u1ncGiYVYsbc1XHQwq981SSNHqr43c7X+LB53aU/flpEXys43TOW7Gg2qUdxmCXpIJWLJzD+SsX0D84xG979pf9+ee272XVklaDXZIaxaolrdz9V2sq/vxZNz9QxWrG5xy7JGXGYJekzBjskpQZg12SMmOwS1JmDHZJyozBLkmZMdglKTMGuyRlplCwR8QdEfFERNw8SbulEfFUdUqTJFVi0mCPiCuBlpTShcBpEXHmBM2/BDTuG14laQooMmLvAO4sbW8A1o7VKCIuBfqA7VWpTJJUkSLB3gpsK233AEuPbBARM4HPAzeNd5CIuCEiOiOis7u7u5JaJUkFFAn2fYxOr8wd5zM3AbenlHrHO0hKaX1KqT2l1N7W1lZ+pZKkQooE+0ZGp19WA1vGaPNO4BMR8TBwXkR8oyrVSZLKVmQ99nuARyNiOXA5cG1ErEsp/e4OmZTSOw5tR8TDKaW/qH6pkqQiJg32lNKeiOgALgNuSSltBzZN0L6jatVJkspW6A1KKaXdjN4ZI0lqYD55KkmZMdglKTMGuyRlxmCXpMwY7JKUGYNdkjJjsEtSZgx2ScqMwS5JmTHYJSkzBrskZcZgl6TMGOySlBmDXZIyY7BLUmYMdknKjMEuSZkx2CUpMwa7JGXGYJekzBjskpQZg12SMmOwS1JmDHZJyozBLkmZMdglKTMGuyRlxmCXpMwY7JKUGYNdkjJjsEtSZgoFe0TcERFPRMTN4+w/PiIeiIgNEXF3RMysbpmSpKImDfaIuBJoSSldCJwWEWeO0eyDwK0ppXcB24F3V7dMSVJR0wu06QDuLG1vANYCL7yxQUrp9jf8tg3YUY3iJEnlKzIV0wpsK233AEvHaxgRFwILU0pPjrHvhojojIjO7u7uioqVJE2uSLDvA2aXtueO95mIWATcBlw/1v6U0vqUUntKqb2tra2SWiVJBRQJ9o2MTL8ArAa2HNmgdLH0LuAzKaWtVatOklS2IsF+D3BdRNwKXA08ExHrjmjzUeCtwOci4uGIuKbKdUqSCpr04mlKaU9EdACXAbeklLYDm45o81Xgq8ekQklSWYrcFUNKaTejd8ZIkhqYT55KUmYMdknKjMEuSZkx2CUpMwa7JGXGYJekzBjskpQZg12SMmOwS1JmDHZJyozBLkmZMdglKTMGuyRlxmCXpMwY7JKUGYNdkjJjsEtSZgx2ScqMwS5JmTHYJSkzBrskZcZgl6TMGOySlBmDXZIyY7BLUmYMdknKjMEuSZkx2CUpMwa7JGXGYJekzBjsklQj737zMt60bN4xP8/0Y34GSRIA//KB82tynkIj9oi4IyKeiIibj6aNJOnYmzTYI+JKoCWldCFwWkScWUkbSVJtFBmxdwB3lrY3AGsraRMRN0REZ0R0dnd3l1+pJKmQIsHeCmwrbfcASytpk1Jan1JqTym1t7W1VVKrJKmAIsG+D5hd2p47zmeKtJEk1UCRAN7I6NTKamBLhW0kSTVQ5HbHe4BHI2I5cDlwbUSsSyndPEGbP6p+qZKkIiYdsaeU9jBycfRJ4JKU0qYjQn2sNq9Vv1RJUhGRUqr9SSO6ga01P3F1LQF21ruIBmJ/HM7+GGVfHO5o+uOUlNKkd5/UJdhzEBGdKaX2etfRKOyPw9kfo+yLw9WiP7x7RZIyY7BLUmYM9sqtr3cBDcb+OJz9Mcq+ONwx7w/n2CUpM47YJSkzBrskZcZgn8Rk68xHxPER8UBEbIiIuyNiZq1rrKWi6+5HxNKIeKpWddVDGX1xe0S8p1Z11UuB/ysLI+L+0iqvX6t1fbVW+j/w6AT7Z0TEvRHxeERcX81zG+wTKLjO/AeBW1NK7wK2A++uZY21VOa6+19idGG47BTti4i4GFiWUrq3pgXWWMH+uA74duke7nkRke297RGxEPgmIyvfjudGYGNKaQ3w/oio2jvzDPaJdTDJOvMppdtTSj8u/bYN2FGb0uqig8nX5iciLgX6GPlGl6sOJn8HwQzg68CWiHhf7Uqriw4m/7exC3hLRCwAVgAv1aa0uhgCrgH2TNCmg9E+ewSo2jc6g31iRdaiByAiLgQWppSerEVhdTJpf5Smoj4P3FTDuuqhyL+NDwG/Am4BLoiIG2tUWz0U6Y/HgFOATwHPltplKaW0p8CaWYXzpVwG+8QKrTMfEYuA24CqzpM1oCL9cRNwe0qpt2ZV1UeRvjgfWJ9S2g58C7ikRrXVQ5H++ALwsZTSPwDPAR+pUW2N6pi9x8Jgn9ik68yXRqh3AZ9JKTX7wmaTKbLu/juBT0TEw8B5EfGN2pRWc0X6YjNwWmm7neZf+G4iRfpjIXBORLQAbwem+kM0x+49Fiklv8b5AuYDm4BbGfnRcTWw7og2Hwd2Aw+Xvq6pd9317I8j2j9c75rr/G9jHiPf9B8BngBOqnfdde6PC4BnGBmp/hiYW++6a9AvD5d+vRT45BH7Tin1xz8D/8PIxeeqnNcnTydRurp9GfBIGvmRekqzP0bZF4ezP8pXejnRWuBHqYrvsTDYJSkzzrFLUmYMdknKjMEuSZkx2CUpMwa7JGXm/wHZEMRkPtUpLQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(precisions, recalls)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## sklearn中的精确率-召回率曲线"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import precision_recall_curve\n",
    "\n",
    "precisions, recalls, thresholds = precision_recall_curve(y_test, decision_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(145,)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precisions.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(145,)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recalls.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "()"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "threshold.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD6CAYAAAC1W2xyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xd8VFX+//HXmWRSSKOkASGEJr0IkSZgaIpRFBuy6+IioCDq6rprXV39KosNXfnZAEVURJai4iogoFQVhQSWXgMBEgikkV4n5/fHDRhCIEOY5M5MPs/HYx4ZZs7ceXud+eTk3HvPUVprhBBCuA+L2QGEEEI4lhR2IYRwM1LYhRDCzUhhF0IINyOFXQgh3IwUdiGEcDNS2IUQws1IYRdCCDcjhV0IIdyMpxlvGhwcrKOiosx4ayGEcFnx8fFpWuuQ6tqZUtijoqKIi4sz462FEMJlKaWO2tNOhmKEEMLNSGEXQgg3I4VdCCHcjBR2IYRwM1LYhRDCzUhhF0IIN2NXYVdKhSmlNl7ieatS6lul1M9KqfGOiyeEEOJyVXseu1KqEfAp4HeJZo8A8VrrF5VSy5VSi7XWOY4Kec6pPbD7a+j9APhXe46+EG5jf0oOy3acMDuGcIDoqMYMuqp265c9FyjZgLuBby7RJgZ4uvz+BiAaWFuxgVLqAeABgMjIyMvNaUjdBxtehy63S2EX9cqsDQl8tTUZpcxOIq7U5OvamF/YtdbZAOrSnyg/ILn8fgYQVsV2ZgOzAaKjo2u2gvbZDLIAt6hn8otsXBXmz6q/Xmd2FOECHHXwNBfwLb/v78DtVnL2l4sUdlG/FJba8LF6mB1DuAhHFeB4YED5/e5AooO2e76zPfayUrBd5Ca9eeGGCkts+HhKYRf2uexJwJRSQ4BOWut3Kzz8KbBcKTUQ6AT85qB857NYjZ+zBl28TUgHmPIrMhgp3ElhSRkBPqbM2SdckN2fFK11TPnPNcCaSs8dVUoNx+i1/1NrbXNkyHNaXwfXT4WSwqqfT90Lu740DrKGdqyVCEJU5Wh6HiU2jbenBR+rBz5WC96eHlg9VHXHp+xSWGIjNMDbAUlFfeCwLoDW+gSwyFHbq5KXH/R/5OLPZyYahf3IRinsok5orXn1+33MWn+4yucDvD25o1cETYN8CPCx4u/jSWiAN31aNb6sgl9YImPswn7u9bddw5YQ1AISN0KfB8xOI9ycrUzz3NJdLNh8jDHXtKB/22AKS2wUldgoLCmjsMTGxkNpfLop8YJDPwHenkQF+xHZpAEtGzegWUNfAnw8y29W/L2N+8H+3vhYPSgsKcPHKheKC/u4V2FXCqIGwv5l8ONLZqcxT6tB0DrG7BRurcRWxuOLtvPt9hM8NLgNf7++fZU98EeGtqOsTFNQYiOnsJScwhLij2ay60QWR9Pz2ZWcxfe7UrCVVX3Q38vTQt/WTcjML5Yeu7CbexV2gM6jjKtTf55hdhJz6DL45V14JA4a1vBCMHFJhSU2pszfypp9p3n6xg5Mvq7NJdtbLAo/b0/8vD0JD/KhXVjAec+X2spIzysmp7CU3CKj+OcWlpJTWMq+lBzWHThNUWkZ4UE+tfmfJdyI0iacHhgdHa1labxakpUE7/SCjrfAHR+ancYlZBeWsG5/Kqt2p5CZX8xbo3sQFlh1Ec0pLGHip3FsTsxg6qgu3NOnZZ1kTM0polEDK54eMhxTnyml4rXW0dW1c78ee30XFAH9HoKNb0LfB6F5T7MTOaVT2YWs3nOKVXtOsSkhjRLb7x2ca19dQ2STBkQ2bkDzhr4E+3sTHOBNsJ8XH6xPYM+JbN6+uwe39mheZ3lD5IwYcRmksLujax+D+E9h1XPwp68u85x+BZ5etRbNTIdO57JqTwqrdp/if8fPANCySQPuu7YVN3QOo0eLRhzLyOerrUkcOp3LsYx8th8/Q2Z+yblteHtamDW2F0M7XjBrhhBOQ4Zi3NWWj2DZ32r22pEzoNc4h8YxQ1mZZnvSGVbtOcXK3SkcTs0DoFtEENd3CuP6zuG0C/Wv9rTDElsZmXnFpOYW0djPi6ZBvpdsL0RtkaGY+q7XfeDpA7mnLu9169+A0/tqJ1Mt0Fqz7kAqgT6eRDRqQJCvld+OZLBqdwqr95zidE4RnhZF39ZNGNc/imEdw2jW8PIKs9XDQmigD6EXGXcXwtlIYXdXFg+4+k+X/7rNH0JxruPz1JL1B1K5b+6WCx5v4OVBTPsQru8UzuD2oQQ1sJqQTghzSGEX57M2gJJ8s1PYbdPhdKweig/u6cXJrAJO5xTRo0VDrm0bLOd9i3pLCrs4n5cfnNgGq/8JDZpA70lgdd4hiM1HMugW0ZBhneRgphBnyUmx4nwR10D2CfhtllHc17xsdqKLyi8uZWdSFr1bNTY7ihBORQq7ON/Nb8Fzp4xbr/tg03twrHZmYb5SW4+eobRMS2EXohIp7OLihr9kXPD0zUNQUmB2mgss23kCX6sHvaOksAtRkRR2cXE+gXDLO5B+ENZMheK8Km7mHGgtKLbx3faTxHZtip+3HCoSoiL5RohLazO4fEjmXeNWlZhnIeapOo21cncKOUWl3Nkrok7fVwhXIIVdVG/EKxDeterz2/cth1/fM+an8favs0hL4pOIaORLHxlfF+ICUthF9ay+cM2Eqp+L7AdzhsP/vqizxU1KbGX8kpDG/YNaY7HI2rZCVCZj7OLKtOhtnCL563tQVjtL3VZ2OqeIMg1RTfzq5P2EcDXSYxdXrt/DsPjP8PVk8A+t+XYsntBnMgQ2vWSzlCxjMfNwmbtFiCpJYRdXruNIiOgN+5fXfBtaQ0keBDSFvpMv2fRsYb/YYhhC1HdS2MWVs3jAxNVXto2SAvhX+CXnqSkr03y9LZnXvt+Hj9VCRGOZPleIqkhhF87Bs7z3fZELobYdy+TFb/ew/fgZukcEMXNsLwJ9ZMZGIaoihV04hcLSMizKm+wzWQRXeDwlq5DXv9/HV9uSCQnwZvpd3bn96uZyNowQlyCFXTiFuT8ncmeZN7u3/crM1F/o0rwhXp4WPvklkVKbZkpMG6YMbou/XGUqRLXkWyJMl5lXzPvrDpFru5EnPBeSmL2YV45fT1FpGSM6h/NsbEcimzQwO6YQLkMKuzDdO2sOkVdUyq2PTocNRYzb+wlj740lo+kgQgK8zY4nhMuRC5SEqeKPZjDv10RGR7fgqvBAGPU+hHbC46sJhBQnmR1PCJckPXZhmqPpedz/WTzNG/ry1IgOxoNefjBmPsweDPPvgnbDq9+QT5BxkZRPYO0GFsJFSGEXpsjKL+G+T7ZQpjVz7+tNIz+v359sFAWjP4WvH4TtC6rfWGE2nDkGt82stbxCuBIp7KLOlZVpJn8ez/GMfD6f0IdWwVXM+dJqEDy+274Nrp0G61+D9rHQ6RbHhhXCBdk1xq6UmqOU2qSUeu4izzdSSi1XSsUppWY5NqJwNzuSs9h0OJ1nYzvSp3WTK9/goCegaXf47jHIPX3l2xPCxVVb2JVStwMeWut+QGulVLsqmo0F5muto4EApVS0g3MKN7LhQCpKwS3dmzlmgx5WuG02FOXCt49CfkbNbyWFjskkhInsGYqJARaV318FDAAOVmqTDnRRSjUEWgDHHRVQuJ8NB1Lp0iyIJv4OPJUxtAMMewFWPguvt6r5dvxC4PG9xi8LIVyUPYXdD0guv58B9KyizU/ATcBfgL3l7c6jlHoAeAAgMjKyJlmFi7OVaWb8eJC4o5k8OrSqP/yuUJ8Hjdkh81Jr9vojG2Dfd8Z8NVLYhQuzp7DnAmen0fOn6uGbF4DJWutspdTjwH3A7IoNtNazzz4WHR2ta5xYuKTUnCIeW7iNnw+lc0fPCCZf18bxb2KxQJfba/56rY3CXlbquExCmMCeg6fxGMMvAN2BxCraNAK6KqU8gD6AFG5xzm+H07np/20kLjGT1+/oxpuju+Pr5WF2rAt5lPdzbMXm5hDiCtlT2JcCY5VSbwGjgd1KqamV2ryC0RvPAhoDdpx8LOqDQ6dz+ONHv+Hn7cnSh65l9DUtzI50cZby4Rdbibk5hLhC1Q7FlA+vxADDgde11inA9kptNgOdayWhcGm/Hs7AVqb59L7ezj+Rl1f5+fTFuebmEOIK2XUeu9Y6U2u9qLyoC2G3PSezCfK10sIVVjsKKF9rNeekuTmEuEIyCZioVbuTs+jYNAClXGBhjIBw42eO9F+Ea5PCLmpNTmEJu05k06tlI7Oj2Ed67MJNSGEXtWZLojG+3r9NcPWNnYFXA2OmyKzk6tsK4cSksItakZSZzwfrEvDytLhOjx2gSTtIO2B2CiGuiMzuKByqxFbGnJ+OMOMHY9aJl27pjI/VCc9Zv5jQjrB/hdkphLgiUtiFw8QfzeAfX+9iX0oOwzuF8eItnWne0AXOhqkotBNsmwe5qeAfYnYaIWpECrtwiP0pOdw1cxPhgT7MHtuL6zuHmx2pZkI7Gj9P7wH/68zNIkQNSWEXV2TrsUxC/L35cd8pyjR8/dC1hAX6mB2r5pr1AGsDiPsYWkthF65JCruosR/3nmLiZ3F4KEWQr5UO4QGuXdQBfBvBtY/BumlwdBO07Gd2IiEum5wVI2pk78ls/rJgG52bBfLHPpHkFJYyvFOY2bEco//DENAMVj4DZWVmpxHiskmPXVy21JwiJn4aR4CPlTl/voawQB+eGtEBb0836Sd4+RmLdnw9CXYugu5jzE4kxGVxk2+iqCuFJTYemBdHRl4xH/05+tzQi5+3J54ebvRx6joaml0Nq56DzESz0whxWdzomyhqm9aaJ5bsYNuxM/z77h50aR5kdqTaY7HAbbOMudk/vwPy0s1OJITdpLALu/2SkM6320/wxA3tGdHFRU9nvBwh7eEPC+HMcVhwNxTnm51ICLtIYRd2O5yWB8CdvSJMTlKHWvaDOz6CpDhYMh5ssmyecH5y8FTY7eSZAjwtimB/b7Oj1K1Ot0DsG7D87/D5bdCw5YVtrA2Ms2kaykLtwnxS2IXddiZn0SbEHw+LC8yt7mi974eiHNjyEaQduvD5/HTYsdAYl28/ou7zCVGBFHZhlxJbGfFHM+vXMExlAx83blVJT4DFfzbG4vs/AkNfAA9r3eYTopyMsYsqaa05kpZHWZkG4Ic9p8gvtnFtWxeZW72uNWkDE36A6AnwyzswN9Y46CqECaTHLqq0cvcpJn8eT7MgH27r2ZzVe07ROsSPYR3d5OrS2mD1gZvfgqhr4b+PwqyBMGqmDM2IOic9dlGlxXHHCfb35qrwAD5Yl8CBU7k8OrRd/Rxfv1xd7oBJ6yEoAhaMMYZphKhD0mMXF0jJKmT9gVQmDGzFMzd25HR2IbtOZDG4fajZ0VxHkzYQOx0+vgEyjxj/FqKOSGEXF5i53uhh/qmPcVpfaKAPQ1x91kYzeAcaPwuzzc0h6h0ZihHnScst4ovNx7izVwQtGjcwO45r8ymfcqFICruoW9JjF+ccS89n0BtrAbi3X5S5YdyBj/TYhTmkxy4AOJyay+hZm879u2PTABPTuAkvf1AW6bGLOic9doHWmns/3kyJrYy/DG1H52aBKCVnv1wxpcA7AAqzzE4i6hkp7ILsglKSMgt47qaOTBzY2uw47sU7SIZiRJ2Twl6PZRWUsHJXCl9tSwIgJKCeTe5VF3wCZShG1Dkp7PVMqa2MVXtO8c3/klm7L5ViWxlRTRrwl6Ht3GfNUmfSoAmc+B9kn4TApmanEfWEFPZ6ZsHmYzz/zW5CArz5U9+W3NqjGd0igmRMvbYMeR7mjYJPR8K47yCgHixQIkwnhb2eScstBmDT00Pca41SZ9XiGrhnibG83qcjYdwy8JcreEXtsuubrZSao5TapJR6rpp27yulRjommqgNCam5NGpglaJel1r2g3sWQ1aSUdxzU81OJNxctd9updTtgIfWuh/QWinV7iLtBgLhWutvHZxROEBRqY0VO0+yavcpbu3R3Ow49U/UtfDHRZB51CjuJQVmJxJuzJ6hmBhgUfn9VcAA4GDFBkopK/AhsFwpdavW+pvKG1FKPQA8ABAZKcuH1QWtNVuPZfLl1mSW7ThJVkEJ4YE+jOsfZXa0+qnVQBg5A75+wDig2rKf2YmEm7KnsPsByeX3M4CeVbS5F9gDvA48opSK1Fq/U7GB1no2MBsgOjpa1zixuKRDp3NJPlPAwLbBTPwsjjX7TuNjtXBD53Buu7o5A9oGyzCMmUKuMn4WZJibQ7g1ewp7LuBbft+fqodvrgZma61TlFKfA/8C3qminahFWQUl/Omj30jJLqRNiB8JqXmMjo7gnyM74+8tx8mdgm9j42e+FHZRe+zpusVjDL8AdAcSq2hzCDh7yWI0cPSKk4nL9tK3e0jNLeLpGzvg7ekBQGzXplLUnYlvI+NnQaa5OYRbs+cbvxTYqJRqBtwIjFFKTdVaVzxDZg7wsVJqDGAF7nR8VHEpq/ec4sutSTwypC2Tr2vDpEGtOZ5RQIvGvtW/WNQd7wDw9IHTe81OItxYtYVda52tlIoBhgOva61TgO2V2uQAd9VKQlEtrTVTl+2hQ3gAjwwxTlpSShHZROZTdzpKQc8/w5YPof8jENbJ7ETCDdl1FE1rnam1XlRe1IWT2ZSQztH0fCYMaIWXpxwYdXoxTxurK618FrScRyAcT6qAi9uXks2UL7bSvKEvI7rI5eouoUFjo7gfXgsHV5mdRrghKewu7tmvduLlYWHB/X0J8LGaHUfY65qJ0KQtrPwH2ErNTiPcjBR2F5aQmsvO5Cxu7xkh4+muxsMKQ/8J6Qfh4Eqz0wg3I4XdRS3bcZJb3vkJf29PbrtapghwSe1vgoBmEPex2UmEm5HC7oK++O0YD32xlfbhASx/dCDtw2V9Upfk4Qm9/gyHfoSMI2anEW5ECruLWbv/NM9+vZPB7UP44v6+NA2S89RdWs97jQWv4z8xO4lwI1LYXcip7EKmfreHVsF+zBzbCx+rh9mRxJUKbAbtb4TNs2HnErPTCDchhd1FLI47zrA315OUWcALIzudmzJAuIHY6RDeDb6cAN/9FUoKzU4kXJwUdheQkVfMk1/u4KrwAFY+NoiY9rICj1sJbGosm9f/L8aB1DnDIT3B7FTChUlhd2LFpWV8v+skjy38H1rD2L4tiQr2MzuWqA0eVrj+ZfjDf+DMMZgdA3suWNZACLtIYXdSO5Oy6P/qj0z+fCv7U7J5eHBbhncKMzuWqG3tb4RJGyC4HSy6F9ZOMzuRcEEyn6sTKrGV8fJ3ewD4eFw0g9qFyOIY9UmjlnDf98aY+0//hn4Pg0+g2amEC5Fq4US01uw9mc24uZvZnJjB365vz5AOYVLU6yNPL+g7BWzFMp+MuGzSY3cCe05k883/kvl+dwpH0/OxeijeuLMbd0W3MDuaMFOL3uAXAvu+g66yxIGwnxR2k63ec4oHP48HoH/bYCYNasPwTmGEBHibnEyYzuIBHW4yzm8vKQSrj9mJhIuQwm4SW5lm5voE3v7hAJ2bB/HJuGto5OdldizhbDqMNK5KPbIBrrre7DTCRcjgrQnyikoZN3czb6zcz/WdwvlsfG8p6qJqUQPA0xcO/WB2EuFCpMdugkcWbOPnQ2m8cntXxlzTAqWU2ZGEs7L6GMVdCru4DNJjr2M/HUxjzb7TPDbsKv7QO1KKuqhe22GQkSAzQAq7SWGvQ5uPZHDfJ5tpF+rP2L4tzY4jXEXbYcbPRfdC2kFzswiXIIW9Dn248TABPlaWPNhfxtSF/Zq0ga53Qeo+eDca5o+Gw+tkIWxxUVLY68i8X4+yes8p/twviiBfWZtUXAal4I6P4K97IOYZOLEVPrsVZg6AbfOhtMjshMLJSGGvA9/tOMHzS3cxpEMoDw9pa3Yc4ar8QyDmaXhsF9zyLugy+GYK/LsLrH8d8tLMTiichNIm/DkXHR2t4+Li6vx9zZCUmc8N/95Ah6aBzJ/YRxbHEI6jtTEks+k9OLQaPH2g22jo9wiEXGV2OlELlFLxWuvo6trJ6Y61qLi0jGe+2olNa2aM6SFFXTiWUtBmsHFL3Q+/vg/b/wM7v4THdoBfsNkJhUlkKKYWzVyfwMaDabw4sjMRjRqYHUe4s5D2MHIG3L8GSvJkDdV6Tgp7Ldl7MptFccdpG+rPmN6RZscR9UVYZ2h1nbESk63U7DTCJFLYa8GWxAzumrmJ4tIypt3W1ew4or7pMwmyk2H/crOTCJNIYXegrPwSvt6WxPhPthAa6M1/Hx5A71aNzY4l6purRkBQJGyebXYSYRI5eOoACam5vPTtHn4+lEZpmaZlkwZ8Nr434UEyzaowgcUDutwOm96F0mJj0Q5Rr0hhr4HcolLiEjPYejSTrcfOsDkxgwZeHtw/qDXDO4XRI6IhFovMASNMFN4Vykoh7QCEdzE7jahjdhV2pdQcoBOwTGs99RLtwoDvtdZXOyif01mz7xRPLtlBWm4xFgUdwgP5Y+9IpsS0ITRQeujCSYR2Mn6e3iOFvR6qtrArpW4HPLTW/ZRSHyul2mmtLzYT0XTA16EJnUBBsY3NiRl8u/0ES+KT6BAewFuje9CzZSP8veWPHuGEmrQFixVO7TY7iTCBPVUpBlhUfn8VMAC4oLArpYYAeUCKo8I5gxk/HOS9tYcotpXh5WHh/oGt+PsN7fH2lIuNhBPz9IKwTrBjoTGBmPTa6xV7CrsfkFx+PwPoWbmBUsoLeB64DVha1UaUUg8ADwBERjr/ed2nswt5+8eDfPHbMdqHBfBMbAf6tGqCr5cUdOEibnkXvhgNH4+Au+ZCu+FmJxJ1xJ7THXP5fXjF/yKveRp4X2t95mIb0VrP1lpHa62jQ0JCLj9pHUrNKWLCp3EsiU/iD71bsOCBvsS0D5WiLlxL027GlaiNWxkFfvOHZicSdcSeHns8xvDLr0B3YH8VbYYBQ5RSDwE9lFIfaa0nOi5m7cotKmXV7hR2JWez60QWO5OysJVp/t8frmZEl3Cz4wlRc4HN4L4V8OVEWP53SD8EN0wzTokUbsuewr4U2KiUagbcCIxRSk3VWj93toHWetDZ+0qpda5U1AH+smAba/adxsdqoVPTQEZHRzC2XxRtQ/3NjibElfP2hzHzYdXz8Ot7cOY43P05WOT6RHdVbWHXWmcrpWKA4cDrWusUYPsl2sc4LF0dOJNfzLr9p7m5W1PevrsHnh7yYRduyOIBI6YZPfhV/zBmguz/sNmpRC2xq4pprTO11ovKi7rb0Frz8Bfb8LRYmDCglRR14f76PQTtb4IfX4LTe81OI2pJva1kvySkMf6TLfx0KI3J17Xm6shGZkcSovYpZUzv6x0AXz1gTDkg3E69K+yFJTY+/SWRez76jS2JmYzt25I/948yO5YQdcc/xCjuKTtgw+tmpxG1oF5cNllqK+NQai67krP55Jcj7ErOpltEELPG9qJpkNtdKCtE9TreDD3ugY1vQlgX6DzK7ETCgdy+sP96OJ1HFmwjNcdYyb2BlwfTbuvKH3q3QCmZqEvUY7FvGKc/fjnROHOm7TCzEwkHcavCnldUSkp2IeGBPny1NYntSVksiU8iNMCbt+/uQZfmQbQK9sNDZl4UArz84I+L4JObYeFYGLsUIvuYnUo4gFsV9smfx7PxYNq5fzfx82Jw+xDuH9Sa/m1kYV8hLuDbEMZ+ZUw7MP8uuG+ZMeWvcGluU9hX7zl1XlF/dGg7HhvWToZbhKiOfyjc+w18fAPMuw0e/MV4TLgsly/stjLNqt0pPDh/K62C/Vg0qR95RaVEBfuZHU0I19GwBdz5sVHcE38yVmASLstlC3tOYQkvf7eHFbtSyCkspVmQD59P7ENIgDchAd5mxxPC9YSVT+2becTcHOKKuWRhP5NfzF0zN3E4LY87ejZnQLsQhncMk9kXhbgS3v7gFwoZUthdnUsV9n0p2cxcl8CynSexlWnmjLuGwe1lLFAIh2kUBZmJZqcQV8ilrjz96WAaS/93ghKb5u0xV0tRF8LRwjrBsU3w48tQUmB2GlFDLlXYB10VQqCPJ/Mm9OaW7s3MjiOE+xn6AnS5EzZOhw/6Q8JasxOJGlBa6zp/0+joaB0XF1fn7yuEsNPhdfDdXyHjMHS721icw0+uBTGbUipeax1dXTuX6rELIepI6xh4cBMMehJ2fQXvRsPWz6CszOxkwg5S2IUQVbP6wJB/wIM/Q0hH+O8j8OlIyEur/rXCVFLYhRCXFtIexi2DW96B5Dhj+oGsJLNTiUuQwi6EqJ7FAj3vhT99BbmnYM4NkHrA7FTiIqSwCyHsF3Wt0Xu3FcHcEZC81exEogpS2IUQl6dpNxi/Eqx+xpj74fVmJxKVSGEXQly+Jm1gwkoIagHz74Qdi8xOJCqQwi6EqJnAZjB+BUT0hq/uh/VvgAnXxYgLSWEXQtScbyNjoY5ud8PaqfDNw2ArMTtVvedSk4AJIZyQpzfcNsuYQGz9a5B1HEZ/ZqzOJEwhPXYhxJVTCgY/C6M+gKM/G+e6F5wxO1W9JYVdCOE4Pf4I9yyG9IPGlaoy5m4KKexCCMdqMwSG/hP2/hfi5pidpl6Swi6EcLx+j0Db4fD9s5Cy0+w09Y4UdiGE41ksxni7byNYfB8U5ZqdqF6Rwi6EqB3+IXD7bEg/BMufMDtNvSKFXQhRe1pfB4OegO1fwPb/mJ2m3rCrsCul5iilNimlnrvI80FKqRVKqVVKqa+VUl6OjSmEcFnXPQWR/eG7xyHtkNlp6oVqC7tS6nbAQ2vdD2itlGpXRbN7gLe01tcDKcAIx8YUQrgsD0+44yPjQqYl46Ck0OxEbs+eHnsMcHaGn1XAgMoNtNbva61Xl/8zBDjtkHRCCPcQ1Nw4mJqyE9ZNMzuN27OnsPsByeX3M4CwizVUSvUDGmmtf63iuQeUUnFKqbjU1NQahRVCuLD2IyBqICT+ZHYSt2dPYc8FfMvv+1/sNUqpxsA7wPiqntdaz9ZaR2uto0NCQmqSVQjh6gKaypqpdcCewh7P78Mv3YHEyg3KD5YuBp7RWh91WDohhHtp0ATyM8xO4fbsKexLgbFKqbeA0cBupdTUSm0mAD2Bfyil1iml7nZwTiGEO2jQBIpzoLTI7CQQlc6uAAAP5klEQVRurdppe7XW2UqpGGA48LrWOgXYXqnNB8AHtZJQCOE+/EONn9nJ0Li1uVncmF3nsWutM7XWi8qLuhBC1Eyzq42fSXHm5nBzcuWpEKLuhHYCL384/pvZSdyaFHYhRN3x8ISIaCnstcxplsYrKSkhKSmJwkK5Kq02+fj4EBERgdVqNTuKqK9a9IENb8Caf0HbodA82ij4wmGcZm8mJSUREBBAVFQUSimz47glrTXp6ekkJSXRqlUrs+OI+urqsXBkA2ycDhteB+9AaDXIKPJthkKjlmYndHlOU9gLCwulqNcypRRNmjRBrvwVpmrYAsZ/DwWZcHg9JKwxbvu+M55v0tZYhanNUIgaAN7+5uZ1QU5T2AEp6nVA9rFwGr6NoPMo46Y1pB2EhB/h0I+wdR5sng0WK0T2NQp926EQ1tVYxENcklMVdle0adMmunbtyvr167npppsctt24uDhsNhu9e/fGZrPh6Xnx/1WFhYX4+PgAxrEKQMbQhWtRCkKuMm59HzQuYDq2ySjyCWvgx/8zbn4h5UV+GHS+DTzkc14V+dVXgdYarTXDhw+nqKiIEydOcODAAW699VY2bdrE/v372bVrF7NnzwagrKyMcePGUVRUxNq1a1m6dOl52ztbZAFmzJjBf/7z+0IDpaWll8xisViYMGECO3bs4KabbuLmm28mNjaW2NhYrrrqKrZt23au7ahRo1i/fj2JiYnMnTuX8ePHk5iYSEJCQrXvI4RT8vSG1jFw/cvw4M/wt/0waqbx2KEf4Kv7YddX5mZ0YtJjr2DBggXk5eXh7e1NbGwsTz75JMeOHSMtLY3ffvsNHx8fxo4dy/PPP0/Dhg2xWq2kpaVxzz33UFRUxI4dO3jzzTd599136d69O/3798fX1xeLxcKxY8eIjIxk5syZaK3Jz89nw4YN+Pr6VpmlZ8+ezJkzh+7du7Ny5Uquu+46Ro0aRfv27dm8efO5HnxCQgLe3t4UFRWxePFitmzZQlFREUuWLKG0tJSHHnqIgICAutyNQjheQDj0+INxKymAf4VDdpLZqZyWFPYKRo8ezcyZM8nJyeGOO+7A29ubJUuWcOzYMZYtW0bTpk2ZPHky8+bNIz4+nocffpiRI0fyySefMGbMGF599VWioqLObW/EiBH07t2btLQ0EhISaN26NS1atCA5OZmEhISLFvXk5GReeuklzpw5w6lTpwDYuXMnBQUF+Pv7M2jQoHNj5c8++ywdO3Zk2LBhvPLKKyQlJWGxWMjKyuL555+Xoi7cj9UXvIMgV5Z9uBgp7BWsX7+e3377jcTERPLz84mLiyMmJgar1co111yDUorly5cTGxuL1WqlXbt2rF69mhEjRrB9+3buvfderFYrP/74I2AU3cTERF5++WXy8vJo27YtCxcuZPny5bz44osXzXG2Bz5lyhRSU1MJCQlh+vTpPPXUUwD88MMPACxevJjt27fTqlUrLBYLeXl5zJs3D4Bly5aRmZlZuztMCLP4h0LuKbNTOC2nLOz/9+1u9pzIdug2OzUL5IWRnS/ZZujQofzwww+Ehobi6+tLr169eO45Y5nXtWvX0qlTJ2JjY9m7dy9Hjx5l2rRpvPrqqxftse/bt4+FCxfy2muvMWfOHCZNmsT9999Pw4YNOXPmDOPHj2fSpEns37//3GuGDBnClClTAHj33Xd59NFHmT59OoGBgcyYMYN//OMf59p27tyZt99+m3Xr1gFQUFDAoUPGmpKnT0tvRrgx/zA4c9zsFE7LKQu7WTIyMti8eTOhoaFs2LABm81Gly5diImJAeDDDz8EYO7cuQwbNgyANWvWMGrUKOLi4pg4cSJWq5UVK1aQlZXFrFmz+Ne//sXEiRPJyspi+vTpWK1Wpk6dyoIFCyguLmbWrFkX5EhLMxYiKC0t5bnnniM4OJghQ4bw1ltv0aNHj3MHaTt16kR+fv651508eZKPPvoIgJSUFIYPH15r+0oIU7WOgbVT4chGaDXQ7DROxykLe3U969oyb948XnjhBV5//XXmz5/P2rVryczMJCIiAgAPDw8yMjJYvXo1r776KgcOHGDIkCFV9ti//PJLjhw5woQJEwBITU0lJyeH5s2b8+ijj1JSUkJwcDATJ06sMktJSQlTp05l9+7dnDhxgrS0NLy9vZk7d+5Fp10IDg5m1KhRAGzevNnBe0cIJ9L/Ydj2Gax4EiZtlCkJKpG9UcGjjz4KwGuvvYaPjw+lpaUsXryYjRs3AkZh37BhA/fccw8WiwWtNStWrGDAgAEcPnyYu+66C4vFwr333stDDz3E+PG/rxL49ttvEx4ezpgxY6rNUVpaSl5eHm+99RZDhw4lMjKS2bNn88YbbzBv3jxatGhx7uCp1pqysjJsNhtBQUEMGGAsdnXmzBkAbDYbHh4eDt1PQpjO6gs3TIOFf4ItH0HfyWYncipS2KtwdnijpKSEp59+mnHjxvH2229z4MCBcz1iMIrmjTfeyCeffFLtNouKiuw+pzwvL4+GDRsyZ84cwDhffujQoQQFBbFq1SrWrFlz7kBqUVERaWlpxMbGEhISct5B2S1btlBaWmrXLxMhXE6Hm6H1YFg7DbrcAf6ylvJZSmtd528aHR2t4+LOn2h/7969dOzYsc6zVKW4uBgvL6/zHnOWnq8jcjjTvhbiiqQegA/6QfQEiH3d7DS1TikVr7WOrq6dXHlahcpFHXCKog7Ok0MIpxByFXQbA1s/lfPaK5DCLoRwbQP+CrZi2PSu2UmchhR2IYRrC24LnW+HLXMgP8PsNE5BDp4KIVzfwL/BriXw6UgI7woNIyGohTH3e8NICIwAzwuHWN2VFPZKnOUgqUzbK8RlCOsEN7wCe/9rXLSUcwJ0WYUGyphILKi80DdsUX6/5e/3vRqYFt/RpLBX8vjjj7Nr1y4KCgo4ePAg3bt3B4wrOe+8805efPFFWrduTWRkZJWvLyoqYtOmTYBRZM8W2BkzZhAWFnbu1MPS0tJLFmuLxcK4ceOYP38+Tz75JFarlbIy44N66NAhFi5cyNVXXw0Y0/Y+88wztGzZklWrVrFx40ZefvllbDYbLVu2vOT7COE2+k0xbgC2EshONqYdOHMMso6X3z8KyXGw5xsoKzn/9Q2aVOjpR57f6w9qAb4N6/6/qYbkG1/JjBkzAFi6dCkbN27kzTffBOC9994715OPjIw8Nz9LZYMHDz53X6btFcIkHlZoFGXcqlJmg5yU8wv+2fup++DgKiitdIW3d1CFnn7FXn95z79BE2PBECcghb0KWmvef/99pk2bdu6x48ePM3CgMSfF2aI6btw4kpKMOaE7d+7MjBkzzhvGkWl7hXBSFg8Iam7cIvte+LzWkJcGWceMHv+Z4+WFv/zfiT9Bcc75r7E2gKCI83v6jVtD+1hj4ZA65JyFfcXTkLLTsdsM7wo3vlpts/T0dCZOnEi7du2IjjauA3jiiSf4/PPP+dvf/nZe27S0tHNT6J6dKKwimbZXCBellHElq38INO914fNaQ+GZSgX/ePkvguOQvBUKys/QaXcDjJlfp8v4OWdhN8kvv/zC+PHjeeKJJ86b52XUqFFMmTKFkJDLu2RZpu0Vwk0pZSzG7dsImnaruk1RLmz7HL5/Cr55yFjar44W4nbOwm5Hz7o29O/fn127dhEeHs7cuXPPey45OZlXXnnlvHlXbDbbuZ56WFjYee1l2l4h6jlvf2NysuIcWDPV+CUw4tU6GYd3zsJuIk9PTyIiIvjpp5/Oe3z69OnnxtbPnp2yYsWKC15/du4dmbZXCAHAwL9Dfib8+h74NoaYp2r9LaWwVyE1NfWCMfPjx4/z2muvAZCUlHRuoY3Kzhbd8ePHy7S9Qgijh379VCjIhHXToEFj6H1/rb6lFPYqBAcHX3A64/Tp089d/POf//yHnj17VvnayrNWniXT9gpRj1kscMs7xpw2Dau+BsaRZNpeFyPT9gpRfzl02l6l1Byl1Cal1HNX0kZcORlWEUJUp9rCrpS6HfDQWvcDWiul2tWkjT3M+OuhvpF9LIT7s6fHHgMsKr+/ChhQkzZKqQeUUnFKqbjU1NQLNuDj40N6eroUnlqktSY9Pf3chGFCCPdkz8FTPyC5/H4GUNVRw2rbaK1nA7PBGGOv/HxERARJSUlUVfSF4/j4+BAREWF2DCFELbKnsOcCZyc18afqXr49bS7JarXSqlWry32ZEEKISuwpwPH8PrTSHUisYRshhBB1wJ4e+1Jgo1KqGXAjMEYpNVVr/dwl2lQxXZoQQoi6UG2PXWudjXFw9FdgsNZ6e6WiXlWbLMdHFUIIYQ9TLlBSSqUCR+v8jX8XDKSZ+P72kIyO4QoZwTVySkbHqWnOllrraqeZNaWwm00pFWfP1VtmkoyO4QoZwTVySkbHqe2cdTM5sBBCiDojhV0IIdxMfS3ss80OYAfJ6BiukBFcI6dkdJxazVkvx9iFEMKd1dceuxBCuK16U9iVUmFKqY0V/t1cKZWklFpXfru8larrOaWUp1LqWIX919XsTK5G9qFjVPxuy/faUC8Ku1KqEfApxmRlZ/UB/qW1jim/OcXsY1X8ArIqpb5VSv2slBp/qdfWsW7Aggr7b6fZgSpzgTUCXGEfViyaTvdZrOK77XTfa6VUkFJqhVJqlVLqa6WUV21/NutFYQdswN1AdoXH+gITlVJblVLTzIl1vov8AnoEiNdaXwvcqZQKMCXchfoCNyulNpd/SJ1qmUVHrRFQy5x9H1b+PDrjZ7Hyd9vpvtfAPcBbWuvrgRRgDLX82XTLwq6UmlXhT7F1wGNVTHOwAmMahGuAfkqpbnUcsypV/QKK4fe57jcAplx8UcU+DQGGaa17A1Yg1oxclxBD9esImG0Lzr0PK38eY3CCz2JFWuvsSt9tp/tea63f11qvLv9nCPAnavmz6VQ9BEfRWk+yo9kvWusiAKXUNqAdsKNWg1WilJoFtK/w0Bqt9UtKqYrNKs91H1ZH8c5TeZ8qpbzP7j8gDmP/ORN71hEw2w5n3oflc0BR4fPoFJ/Fapj+vb4YpVQ/oBHG7Le1+tl0yx67nVYqpZoqpRoA1wO76jqA1npShbHAGK31S1U0u+K57mvJPKVUd6WUBzAK2G52oEqcdb9V5Oz7sDJX2Kemf6+ropRqDLwDjKcO9qMz/o+pK/8HrMWYkXKm1nq/yXkuxlnnun8JmAf8D9iktf7B5DyVOet+q8jZ92FlrrBPne57rZTyAhYDz2itj1IH+1EuUHJCSql1WuuY8vstgeXAD0B/oK/W2mZiPJeglAoENgI/Ur5GgEwnXTNnP4/yWawZpdSDwDR+/4tsLvA4tfjZlMLuAsoXMBkArJTiZL/yszqGAxu01ilm53EH8ll0jNr+bEphF0IIN1Ofx9iFEMItSWEXQgg3I4VdCCHcjBR2IYRwM1LYhRDCzfx/D4wDtKY/1IoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(thresholds, precisions[:-1], label=\"精准率-阈值\")\n",
    "plt.plot(thresholds, recalls[:-1], label=\"召回率-阈值\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD6CAYAAAC1W2xyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAFNFJREFUeJzt3XuQXnV9x/H3N5sNCbsJ5LLZNEAIwQAFbBRXEAx2SYmXtmoHLeJ0sIPaaIdip9brFGvR9CJTmWntwJhKK6OOFq2lxluxxQhVUDdqjKCxIRchySa7bNjNbrK57a9/7BMT1t19zrN5bnv2/ZrJcML+nnM+nFk+e/b3nPP8IqWEJCk/ptU6gCSpvCx2ScoZi12ScsZil6ScsdglKWcsdknKGYtdknLGYpeknLHYJSlnptfioAsWLEhLly6txaEladLauHFjd0qppdi4mhT70qVL6ejoqMWhJWnSioidWcY5FSNJOWOxS1LOWOySlDMWuyTljMUuSTljsUtSzmQq9ohojYhHxvl6Y0Ssj4hvR8SbyxdPklSqosUeEXOB+4CmcYbdBmxMKb0UeH1EzC5TvlEdOnKcv/rS4zzVc7CSh5GkSSnLFftx4A1A3zhj2oH7C9sPA20jB0TEmojoiIiOrq6uUnM+x+e+/ws++Z0d7H720GntR5LyqGixp5T6Ukq9RYY1AbsK2z1A6yj7WZdSaksptbW0FH0idkxHjg2x7uFtvHjpXK5aNn/C+5GkvCrXm6f9wKzCdnMZ9/srvviDp9nTO8it1z2vUoeQpEmtXJ8VsxFYCXwBWAE8Vqb9Psex40Pc860nWdbSxPLW2SVPxSxoPoMZ070RSFK+lVzsEbEKuDSl9E+n/Ov7gK9GxLXApcB3y5TvOTY93cvOZ4bfMH3p3z1U8uuv//WFfOIPX1zuWJJUVzIXe0qpvfDPh4CHRnxtZ0SsZviq/S9TSsfLGfKE559zFh974ws5eORYSa/76Z4DfPI7O7j8nLMqEUuS6krZPrY3pbSbk3fGVMSM6dN49YrFJb/uln/9Hmef2cibV15QgVSSVF9yP+G8cWcP39zSxdtediFzZjbWOo4kVVxNFtqopo89tBWAh362l+882V3jNCrVqksWcstL/U1LKkXui/3yxWfRe+gox4YS/YdLm5tX7RwYPMbWff3MmdVosUslyn2xv+sVF/MuLq51DJXo/V/czI7uAd65+qJaR5EmndzPsWvyeWJ3H//2/V/wpquXcmFLc63jSJNO7q/YNbmklPjwl58gActbm/nq5j0VP+bzFjZzUWtFP7dOqiqLXXVl8OgQHTt7SGl4OqYazp9/Jt9693VVOZZUDRa76sqsGQ18+72r2H/waMWP9b0dPXzggZ9wY9t5FT+WVE0Wu+rOwjkzWThnZkWPMTSUeM8XNrFozkze7F03yhnfPNWU9OXNe9j0dC/vesXFzJrRUOs4Ull5xa4p6Z4NTwJwx5ce50PrH8/8upmNDfzrLS/mssV+7pDql8WuKemtKy9g865i68c819d/0kl3/2Fmn+FHU6i+Weyakl73onN53YvOzTx+675+PvXYTm5+yfksmX9mBZNJp885dimDj3z9Z8xqbOC2Va7cpfrnFbtURMeOHr7xxF6WL2zm4w9vq/jxbrjiHC5ZNKfix1F+WexSEX2DR5l9xnSe3n+ITz26s2LHOT6UOHJ8iKYZ0y12nRaLXSpi1SWtbL7jFRU/zrs/v4n/3LSbm670gSmdHufYpTqwo3uAL/5wF39w1RJaK/xwlvLPK3apDnzsoa2klHjFZYvY1tVf6zhlt3R+E9OmRa1jTBkWu1RjKSX+80e7GEpw07rHah2nIm5b9Tz+/OWui1AtFrtUYxHB59a8hF3PHqp1lLJ7YncfH394GxcsaKp1lCnFYpfqQNvSebTVOkQF/GDnfs6YPo3Vl7bWOsqU4punkiri2PEhvrJ5D6suWcjsmX4MQzVZ7JIq4rFtPXT3H+E1KxbXOsqUY7FLqoj1m3bTfMZ0rrtkYa2jTDkWu6SyO3zsOF/7yR5efmkrMxv9vPtqs9glld0jP++mb/AYr36B0zC14F0xksruS5t20zAt6Oo7zBd/8HSt4wAwLYLfvKiFuU0zah2l4ix2SWW3s+cgx4cS7/n3H9c6ynP82fUX8afXL691jIqz2CWV3Wf/6Cq6DhyudYxf2rqvn7fc18GS+bNqHaUqMhV7RNwLXAp8JaW0dpSvzwU+AywENqaU3lbWlJImlTNnTOf8+fVz3bhx534ALp8ia9UWffM0Im4AGlJKVwPLImK032NuBj6TUmoDZkdEHh+ikzRJ/WRXHzMbp7GspbnWUaoiy4/UduD+wvaDwErg/0aMeQa4PCLOBs4DnipXQEk6XVu7+hk8OsSvf+DrRcfOmD6NT97yYtqWzqtCssrIUuxNwK7Cdg9wxShj/hf4HeAdwE8L454jItYAawCWLFkykaySNCFv/81lXLa4+KpUT+8/xPpNuzkweKwKqSonS7H3AyfecWhm9OmbDwJvTyn1RcQ7gVuAdacOSCmtO/Hv2tra0oQTS1KJrrlwAddcuKDouM93PMX6Tbsn/adRZnlAaSPD0y8AK4Ado4yZCzw/IhqAqwCLW9Kks617gMaG4Ny5k/vumSzF/gBwc0TcBdwIPB4RI++M+VuGr8Z7gXnAZ8uaUpKqYFtXP0vmncn0hsn9UH7RqZjC9Eo7sBq4M6XUCWwaMeZ7wGUVSShJVbK9eyAXd85k+rGUUtqfUrq/UOqSlDvHhxI7njnIskk+vw5+CJgkAbD72UMcOTbEshaLXZJy4cmufgAuWDBFpmIkKe+2dw8AeMUuSXmxrWuA2TOnMz8HH+trsUsSsK27n2UtzUREraOcNotdkoDtXQO5uCMGLHZJ4tCR4+zuHbTYJSkvtu4bviPmwoWT/44YsNgliS17DwBw8aLZNU5SHha7pClvS2cfM6ZPY+l8p2IkKRe27O1n+cJmGqZN/jtiwGKXJH7eeYCLW/MxDQMWu6QprvfgUTr7BnMzvw4Wu6Qp7sQbpxflqNizLI0nSbl1otiXL2xmaCj74m8R1O1Tqha7pCltZ+HDv1Z+5Jslve51V5zLR29cUYlIp81ilzSlvfGqJcyZ1UgqYaXmTz22g+7+w5ULdZosdklT2oUtzbzjt5aX9JpPf3cnv3bWzAolOn2+eSpJJThybIju/sO0zrHYJSkX9h0YJCW8YpekvNjbNwhAq8UuSfmwp3e42L1il6Sc6DxR7HNm1TjJ2Cx2SSpBZ+8gMxunMWdW/d5UaLFLUgk6+wb5tbNm1e1Tp2CxS1JJOnsHaZ1zRq1jjMtil6QSnLhir2cWuyRlNDSU2Ns3yKI6viMGLHZJyuyZgSMcPZ5YVMdPnYLFLkmZ7Xr2EACLz3YqRpJy4en9BwE4b14Oij0i7o2IRyPi9iLj7o6IV5cnmiTVl6d6hq/Yz517Zo2TjK9osUfEDUBDSulqYFlEjPr5lhFxLbAopbS+zBklqS48tf8gc89spPmM+n04CbJdsbcD9xe2HwRWjhwQEY3APwM7IuK1o+0kItZEREdEdHR1dU0wriTVzlM9BzlvXn1frUO2Ym8CdhW2e4DWUca8CXgCuBO4MiJuGzkgpbQupdSWUmpraWmZaF5Jqpld+w9xXp1Pw0C2Yu8HTrxT0DzGa14IrEspdQKfBq4rTzxJqg9DQ4mn9x/i3Dp/4xSyFftGTk6/rAB2jDJmK7CssN0G7DztZJJUR/YdOMyR40N1/8YpZFvz9AHgkYhYDLwKuCki1qaUTr1D5l7gXyLiJqAReH35o0pS7Tx14lbHufV/xV602FNKfRHRDqwG7ixMt2waMeYA8PsVSShJdWB79wAAFyxoqnGS4jLds5NS2s/JO2MkacrZ3j1AY0NwTp0/dQo+eSpJmWzvGmDJvDOZ3lD/tVn/CSWpDmzvHuCCBc21jpGJxS5JRQwNJXY8M8CylvqfXweLXZKK2tM3yOFjQyydb7FLUi5s75o8d8SAxS5JRW3v7gdwKkaS8uLJrgGaZjSwcHZ9L2J9gsUuSUVs6TzA8tbZRESto2RisUvSOFJKbNl7gItbZ9c6SmYWuySNo7v/CD0DR7h4kcUuSbmwpfMAgMUuSXmxZa/FLkm5sqWzj/lNM1jQPDnuiAGLXZLGtaXzwKS6WgeLXZLGNDSU+PnefotdkvJiZ89BDh09ziUWuyTlw+ZdvQBctvisGicpjcUuSWN4fFcvMxqmcdEkejgJMi6NJ0lT0eZdvSxraeLA4NGy7G9aBHObZpRlX+Ox2CVpDNu7B9jTO8iL1v532fZ5x2su4w+vWVq2/Y3GYpekMXz0xhVs3ddftv3dsf4J9vQOlm1/Y7HYJWkM11y4gGsuXFC2/a39yk/Ltq/x+OapJOWMxS5JOWOxS1LOWOySlDMWuyTljMUuSTljsUtSzljskpQzmYo9Iu6NiEcj4vYi41oj4ofliSZJmoiixR4RNwANKaWrgWURsXyc4X8PzCpXOElS6bJcsbcD9xe2HwRWjjYoIlYBA0BnWZJJkiYkS7E3AbsK2z1A68gBETED+ADwvrF2EhFrIqIjIjq6uromklWSlEGWYu/n5PRK8xiveR9wd0rp2bF2klJal1JqSym1tbS0lJ5UkpRJlmLfyMnplxXAjlHGXA/cGhEbgBdExCfKkk6SVLIsH9v7APBIRCwGXgXcFBFrU0q/vEMmpfSyE9sRsSGl9NbyR5UkZVG02FNKfRHRDqwG7kwpdQKbxhnfXrZ0kqSSZVpoI6W0n5N3xkiS6phPnkpSzljskpQzFrsk5YzFLkk5Y7FLUs5Y7JKUMxa7JOWMxS5JOWOxS1LOWOySlDMWuyTljMUuSTljsUtSzljskpQzFrsk5YzFLkk5Y7FLUs5Y7JKUMxa7JOWMxS5JOWOxS1LOWOySlDMWuyTljMUuSTljsUtSzljskpQzFrsk5YzFLkk5Y7FLUs5Y7JKUMxa7JOVMpmKPiHsj4tGIuH2Mr58VEV+LiAcj4j8iYkZ5Y0qSsipa7BFxA9CQUroaWBYRy0cZ9gfAXSmllwOdwCvLG1OSlNX0DGPagfsL2w8CK4H/O3VASunuU/7aAuwbuZOIWAOsAViyZMkEokqSssgyFdME7Cps9wCtYw2MiKuBuSmlx0Z+LaW0LqXUllJqa2lpmVBYSVJxWa7Y+4FZhe1mxvhhEBHzgI8BrytPNEnSRGS5Yt/I8PQLwApgx8gBhTdLPw+8P6W0s2zpJEkly1LsDwA3R8RdwI3A4xGxdsSYtwBXAH8RERsi4g1lzilJyqjoVExKqS8i2oHVwJ0ppU5g04gx9wD3VCShJKkkWebYSSnt5+SdMZKkOuaTp5KUMxa7JOWMxS5JOWOxS1LOWOySlDMWuyTljMUuSTljsUtSzljskpQzFrsk5YzFLkk5Y7FLUs5Y7JKUMxa7JOWMxS5JOWOxS1LOWOySlDMWuyTljMUuSTljsUtSzljskpQzFrsk5YzFLkk5Y7FLUs5Y7JKUMxa7JOWMxS5JOWOxS1LOWOySlDMWuyTlTKZij4h7I+LRiLj9dMZIkiqvaLFHxA1AQ0rpamBZRCyfyBhJUnVkuWJvB+4vbD8IrJzgGElSFWQp9iZgV2G7B2idyJiIWBMRHRHR0dXVNZGskjSpvfKyRVyyaHbFjzM9w5h+YFZhu5nRfxgUHZNSWgesA2hra0slJ5WkSe4f3/jCqhwnyxX7Rk5OrawAdkxwjCSpCrJcsT8APBIRi4FXATdFxNqU0u3jjHlJ+aNKkrIoesWeUupj+M3Rx4DrUkqbRpT6aGN6yx9VkpRFlit2Ukr7OXnXy4THSJIqzydPJSlnLHZJyhmLXZJyxmKXpJyJlKr/rFBEdAE7S3jJAqC7QnEqwbyVZd7KMm9lnU7e81NKLcUG1aTYSxURHSmltlrnyMq8lWXeyjJvZVUjr1MxkpQzFrsk5cxkKfZ1tQ5QIvNWlnkry7yVVfG8k2KOXZKU3WS5YpckZTTpij0i5kXE6ohYUOssklSP6qrYiy2IHRFzgS8DVwLfjIii93NWUtYFvCOiNSJ+WK1c4+Qodn6nR8QvImJD4c/zq51xRJ6s5/fuiHh1tXKNk6PY+f3jU87tjyLi49XOOCJP0f/fIuKrhZXPapq1kKdY3gsi4isR8UhEfLTa+UbJ0xoRj4zz9caIWB8R346IN5fz2HVT7BkXxP4N4J0ppb8G/gu4opoZT1XiAt5/z8kVpmqihPP72ZRSe+HP5uqmPCnr+Y2Ia4FFKaX1VQ34qzmK5k0p3XPi3AKPAP9c5Zi/lPH83gx8pnDP9eyIqNm94hnzfgT4cErpWuDciGivZsZTFS5C72N42dCx3AZsTCm9FHh9RJRtzby6KXYyLIidUvpWSumxiHgZw1ftj1Yv3q9oJ8MC3hGxChgAOqsTa0ztFM/7EuB3I+J7haujTB/rXCHtFMkbEY0Ml+OOiHht9aKNqp2MC7pHxDlAa0qpowq5xtJO8bzPAJdHxNnAecBT1Yk2qnaK570I+EFhex9wVuVjjek48Aagb5wx7Zz8b3oYKNsPznoq9iyLZhMRwfAJ2w8crU60UWVZwHsG8AHgfVXMNZYs5/f7wPUppSuBRuC3q5RtNFnyvgl4ArgTuDIibqtSttFk+v4tuBW4p+KJxpcl7/8C5wPvAH5aGFcrWfJ+AfhgYVrulcD/VCnbr0gp9WVYcKiU75mS1FOxZ1k0mzTsVuDHwGuqlG00WfK+D7g7pfRs1VKNLUveH6eU9hS2O4DxppcqLUveFwLrUkqdwKeB66qUbTSZvn8jYhrDOTdUJ9aYsuT9IPD2lNKHgJ8Bt1Qp22iK5k0prQW+BrwVuC+l1F+9eBOS6XtmIuqp2IsuiB0R742INxX+ejZQy8LMsoD39cCtEbEBeEFEfKI60UaVJe+nImJFRDQAvwdsqlK20WTJuxVYVthuo7QPliu3rAu6Xwt8N9X+AZIseecCzy98P1wF1DJz1vP7I2AJcFcVMp2urP9NpUsp1cUfYA7DRXIXw7/2rQDWjhgzF/gGw/NRd1N4wKpe844Yv2ESnN/LGf5NaDPw15Mg72zg84Xvh0eBc+o5b2Hc3wA31PLclnB+rwQeZ/jK8htAcz3nLYy7A7i51uf3lDwbCv9cBfzJiK+dXzi//8DwNGhDuY5bV0+eFt5JXg08nIZ/va5r5q0s81aWeWsvIhYzfNX+X6n4nHz2/dZTsUuSTl89zbFLksrAYpeknLHYJSlnLHZJyhmLXZJy5v8BBlsvQN8J/AkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(precisions, recalls)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 绘制ROC曲线"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "def TPR(y_test, y_predict):\n",
    "    try:\n",
    "        return TP(y_test, y_predict) / (TP(y_test, y_predict) + FN(y_test, y_predict))\n",
    "    except:\n",
    "        return 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "def FPR(y_test, y_predict):\n",
    "    try:\n",
    "        return FP(y_test, y_predict) / (FP(y_test, y_predict) + TN(y_test, y_predict))\n",
    "    except:\n",
    "        return 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "fprs = []\n",
    "tprs = []\n",
    "thresholds = np.arange(np.min(decision_scores), np.max(decision_scores), 0.1)\n",
    "for threshold in thresholds:\n",
    "    y_predict = np.array(decision_scores >= threshold, dtype='int')\n",
    "    fprs.append(FPR(y_test, y_predict))\n",
    "    tprs.append(TPR(y_test, y_predict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD6CAYAAAC1W2xyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAD2BJREFUeJzt3X1sXfV5wPHvkzcICYQE3KSUkhAEo7ylLR5daAADpStsCAm1BamiE6yK6DrYv7Cl7dYhdcoq1KkTVVNSCY1qE2wqEwXUMNEowMJaZyxsUAa0TdoCIQ4BgimkxH72hy8zcW3fY+f6vvzu9yNFOeT+fP0cHH19cu65x5GZSJLKMavVA0iSGsuwS1JhDLskFcawS1JhDLskFcawS1JhDLskFcawS1JhDLskFWZOKz7psccemytWrGjFp5akjrVt27Y9mdlTb11Lwr5ixQr6+/tb8aklqWNFxM4q6zwVI0mFMeySVBjDLkmFMeySVBjDLkmFMeySVJhKYY+IpRHx8CSPz42IeyPi0Yi4rnHjSZKmqu517BGxGLgDWDDJshuAbZn5lxFxf0TcnZmvN2pITd33n3iBZ3b5JZDaTe+KJZx/St33GB2SKm9QGgKuAv51kjV9wE217S1AL/DDdy+IiLXAWoATTjhhqnNqCl4e3M+f/dN/MTScRLR6Gknvdv0FJ7U+7Jm5DyAmL8QC4Pna9l5g6TjPswHYANDb2+tP0J5BP3jyJYaGk/tuXMPpxy1q9TiSmqxRL54OAvNr2wsb+Lyahvv++wVOPHYBp733qFaPIqkFGhXgbcCa2vYqYEeDnlfjyEwODA2P+2v362+x9acvc9mZy+r9K0tSoaZ8E7CIuAg4LTP//l1/fAdwf0ScB5wG/EeD5tMYQ8PJ7399C8/tHpx03WVnvrdJE0lqN5XDnpl9td8fAh4a89jOiLiEkaP2L2XmUCOH1Kgf79jLc7sH+dTZx3PCkiPGXbN00eGehpG6WMNu25uZLwB3Ner5NL77nniRw+fO4q+uOJ0j5rXkrsuS2pwvcnaQoeHkgf/ZxcWnLjXqkiZkHWr+/bk9PPrTPa0eY1J733ibPYP7+YOzPH8uaWKGveZvN/0vj//iVebMau8rSU5YcgR9vzOzb26Q1NkMe81wwgWn9HDHdee0ehRJOiSeY5ekwnTlEfvQcHJgePigP8v0LgeSytB1YX/r7SHOX/9Ddr++/7ceu/jU97RgIklqrK4L+3O7B9n9+n6u/PD7OKln4UGP+aKkpBJ0Xdif3T1yj/LrLziJU5Ye2eJpJKnxuu7F02deGmTOrGDFMZP93BBJ6lxdc8Q+uP8Atz/8Mx586iVOPHYB8+Z03fc0SV2ia+r2o5+/zNf/7Vl+8fKvufgDv/VzQCSpGF1zxP7O1Y3/8vlzOfN4f6qQpHJ1zRG7JHWLoo/YDwwN85uhkUP1/QeG66yWpDIUG/bfHBjm3L95iD2DB78RaXab3+RLkg5VsWF/68AQewb3c/Gp7+GcE5cAcNT8uZy6zGvXJZWt2LC/Y/VJx/C581a2egxJahpfPJWkwhRxxL7xkZ/z4qtvHvRn77xoKkndpuPD/tqbb/PX33+KebNnMXf2wS+MLpo/l5O9H4ykLtPxYX/nPuo3X3Yq1370xBZPI0mt5zl2SSqMYZekwhh2SSqMYZekwhh2SSqMYZekwnTk5Y6ZyYYtP+PF197yro2SNEZHhv3VX7/NVx94msPmzOKwObM4ZsE8fzC1JNV0ZNiz9vufX/YB/ujcFa0cRZLaTqVz7BGxMSK2RsS6CR5fHBH3R0R/RHyrsSNKkqaibtgj4kpgdmauBlZGxMnjLLsG+G5m9gJHRkRvg+c8yIFhz6tL0kSqHLH3AXfVtjcBa8ZZ8zJwRkQcDbwf+GVDppvAn9z5nwDMme1PQ5KksaqEfQHwfG17L7B0nDWPAMuBG4Gf1NYdJCLW1k7V9A8MDExz3BGvvfk2AH941nGH9DySVKIqYR8E5te2F07wMV8Grs/MrwBPA9eOXZCZGzKzNzN7e3p6pjsvABFw6RnLWDR/7iE9jySVqErYtzF6+mUVsGOcNYuBMyNiNvARRi9ckSQ1WZWw3wNcExG3Ap8GnoyIW8as+SqwAXgNWAL8Y0OnlCRVVvc69szcFxF9wCXA+szcBWwfs+ZHwOkzMqEkaUoqvUEpM19h9MoYSVIb8yZgklQYwy5JhTHsklQYwy5JhTHsklQYwy5JhTHsklQYwy5JhTHsklQYwy5JhTHsklQYwy5JhTHsklQYwy5JhTHsklSYjgv7E796lWdeGmz1GJLUtjou7P+87VcA/O6KJS2eRJLaU8eFPROWLJjHdWtObPUoktSWOi7skqTJGXZJKoxhl6TCGHZJKoxhl6TCGHZJKoxhl6TCGHZJKoxhl6TCGHZJKoxhl6TCGHZJKoxhl6TCGHZJKkylsEfExojYGhHr6qy7LSIub8xokqTpqBv2iLgSmJ2Zq4GVEXHyBOvOA5Zl5r0NnlGSNAVVjtj7gLtq25uANWMXRMRc4NvAjoi4YrwniYi1EdEfEf0DAwPTHFeSVE+VsC8Anq9t7wWWjrPms8BTwHrgnIi4YeyCzNyQmb2Z2dvT0zPdeSVJdVQJ+yAwv7a9cIKP+RCwITN3AXcCFzZmPEnSVFUJ+zZGT7+sAnaMs+Y5YGVtuxfYeciTSZKmZU6FNfcAD0fEccClwNURcUtmvvsKmY3AdyLiamAu8MnGjypJqqJu2DNzX0T0AZcA62unW7aPWfM68KkZmVCSNCVVjtjJzFcYvTJGktTGfOepJBXGsEtSYQy7JBXGsEtSYQy7JBXGsEtSYQy7JBXGsEtSYQy7JBXGsEtSYQy7JBXGsEtSYQy7JBXGsEtSYQy7JBXGsEtSYQy7JBXGsEtSYQy7JBWmo8L+0r63+IfHdnJgaLjVo0hS2+qosPfveAWAs44/usWTSFL76qiwv+NLl5/W6hEkqW11ZNglSRMz7JJUGMMuSYUx7JJUGMMuSYUx7JJUGMMuSYUx7JJUGMMuSYWpFPaI2BgRWyNiXZ11SyPi8caMJkmajrphj4grgdmZuRpYGREnT7L8a8D8Rg0nSZq6KkfsfcBdte1NwJrxFkXERcAbwK6GTCZJmpYqYV8APF/b3gssHbsgIuYBXwRumuhJImJtRPRHRP/AwMB0ZpUkVVAl7IOMnl5ZOMHH3ATclpmvTvQkmbkhM3szs7enp2fqk0qSKqkS9m2Mnn5ZBewYZ83HgC9ExGbggxFxe0OmkyRN2ZwKa+4BHo6I44BLgasj4pbM/P8rZDLz/He2I2JzZn6u8aNKkqqoG/bM3BcRfcAlwPrM3AVsn2R9X8OmkyRNWZUjdjLzFUavjJEktTHfeSpJhTHsklQYwy5JhTHsklQYwy5JhTHsklQYwy5JhTHsklQYwy5JhTHsklQYwy5JhTHsklQYwy5JhTHsklQYwy5JhTHsklQYwy5JhTHsklQYwy5JhTHsklQYwy5JhTHsklQYwy5JhTHsklQYwy5JhTHsklQYwy5JhTHsklQYwy5JhTHsklQYwy5JhakU9ojYGBFbI2LdBI8viogHImJTRHwvIuY1dkxJUlV1wx4RVwKzM3M1sDIiTh5n2WeAWzPz48Au4BONHVOSVNWcCmv6gLtq25uANcCz716Qmbe96z97gN2NGE6SNHVVTsUsAJ6vbe8Flk60MCJWA4sz87FxHlsbEf0R0T8wMDCtYSVJ9VUJ+yAwv7a9cKKPiYglwDeA68Z7PDM3ZGZvZvb29PRMZ1ZJUgVVwr6NkdMvAKuAHWMX1F4svRu4OTN3Nmw6SdKUVQn7PcA1EXEr8GngyYi4ZcyaPwY+DPxFRGyOiKsaPKckqaK6L55m5r6I6AMuAdZn5i5g+5g13wS+OSMTSpKmpMpVMWTmK4xeGSNJamO+81SSCmPYJakwhl2SCmPYJakwhl2SCmPYJakwhl2SCmPYJakwhl2SCmPYJakwhl2SCmPYJakwhl2SCmPYJakwhl2SCmPYJakwhl2SCmPYJakwhl2SCmPYJakwhl2SCmPYJakwhl2SCmPYJakwhl2SCmPYJakwhl2SCmPYJakwhl2SCmPYJakwHRX2ZYsO57Izl7HwsDmtHkWS2lZHFfLs5Ys5e/nZrR5DktpapSP2iNgYEVsjYt2hrJEkzby6YY+IK4HZmbkaWBkRJ09njSSpOaocsfcBd9W2NwFrprMmItZGRH9E9A8MDEx9UklSJVXCvgB4vra9F1g6nTWZuSEzezOzt6enZzqzSpIqqBL2QWB+bXvhBB9TZY0kqQmqBHgbo6dWVgE7prlGktQEVS53vAd4OCKOAy4Fro6IWzJz3SRrfq/xo0qSqqh7xJ6Z+xh5cfQx4MLM3D4m6uOtea3xo0qSqojMbP4njRgAdk7zw48F9jRwnE7gPncH97k7HMo+L8/MuleftCTshyIi+jOzt9VzNJP73B3c5+7QjH326hVJKoxhl6TCdGLYN7R6gBZwn7uD+9wdZnyfO+4cuyRpcp14xC5JmoRhl6TCtG3Yu/Ee8PX2JyIWRcQDEbEpIr4XEfOaPWOjVf0aRsTSiHi8WXPNpCns820RcXmz5ppJFf5uL46I+2t3gP1Ws+ebCbW/sw9P8vjciLg3Ih6NiOsa+bnbMuzdeA/4ivvzGeDWzPw4sAv4RDNnbLQpfg2/xuiN5jpW1X2OiPOAZZl5b1MHnAEV9/ka4Lu167uPjIiOvrY9IhYDdzBy59uJ3ABsy8yPAp+MiCMb9fnbMuw06B7wHaaPOvuTmbdl5oO1/+wBdjdntBnTR4WvYURcBLzByDezTtdH/Z9dMBf4NrAjIq5o3mgzpo/6X+eXgTMi4mjg/cAvmzPajBkCrgL2TbKmj9H/L1uAhn0za9ewN+Qe8B2m8v5ExGpgcWY+1ozBZlDdfa6dbvoicFMT55pJVb7OnwWeAtYD50TEDU2abaZU2edHgOXAjcBPaus6Vmbuq3DPrBlrWLuGvRvvAV9pfyJiCfANoKHn5Fqkyj7fBNyWma82baqZVWWfPwRsyMxdwJ3AhU2abaZU2ecvA9dn5leAp4FrmzRbK81Yw9o1ht14D/i6+1M7er0buDkzp3sTtXZS5Wv4MeALEbEZ+GBE3N6c0WZMlX1+DlhZ2+5l+jfMaxdV9nkxcGZEzAY+AnTDG2xmrmGZ2Xa/gKOA7cCtjPyzbBVwS501i1o9dxP2+fPAK8Dm2q+rWj33TO/zmPWbWz1zk77ORzLyDXwLsBV4X6vnbsI+nwM8ychR7IPAwlbP3aB931z7/SLgT8c8try2z38H/JiRF5gb8nnb9p2ntVeVLwG25Mg/Sae1ppOUtj9VuM/uczer/XCiNcAPsoE/x6Jtwy5Jmp52PccuSZomwy5JhTHsklQYwy5JhTHsklSY/wPxqXq37EAZLgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(fprs, tprs)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## sklearn中的ROC曲线"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve\n",
    "\n",
    "fprs, tprs, threshold = roc = roc_curve(y_test, decision_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD6CAYAAAC1W2xyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADblJREFUeJzt3VGonOWdx/Hvb6OCJGojHlKVRg14E1tiy8Fs2LgcxZR6IYLbrWKx0O6S7dK115ZNcSmBBSnC0sXStFkQWpfViyqCZVNYglY03ROqF9rCFoyWbENPG9vUXsp/LzL2ZE/POfPOZGbOmWe+Hwh54vxn5v844XeePO/7zpuqQpLUjj/b6AYkSaNlsEtSYwx2SWqMwS5JjTHYJakxBrskNcZgl6TGGOyS1BiDXZIac8lGvOk111xTN95440a8tSRNrZMnT/66qub61W1IsN94440sLi5uxFtL0tRK8naXOrdiJKkxBrskNcZgl6TGGOyS1BiDXZIaY7BLUmM6BXuSHUleWufxS5M8n+TlJF8YXXuSpEH1PY89yXbgSWDrOmUPAyer6p+SvJDkmar6/aia1OCeOvEOz712eqPbkLTC7uuu5NF7bhnre3RZsb8P3A+cW6dmAXi6N34RmF9ZkORgksUki0tLS4P2qQE999pp3vzleh+ZpFb1XbFX1TmAJOuVbQU+WB6eBXas8jpHgCMA8/Pz3kF7AnZfeyX/8Xf7NroNSRM2qoOn7wGX98bbRvi6kqQBjSqATwL7e+M9wKkRva4kaUADfwlYkjuB3VX1rxf85yeBF5LcDuwGToyoP11gkAOib/7yHLuvvXLMHUnajDqv2Ktqoff7f60IdarqbeAA8DJwV1W9P8omdd4gB0R3X3sl9956/Zg7krQZjexre6vqf1k+M0Zj4gFRSf14kFOSGrMhN9rYjKbhgh73zSV14Yq9Zxou6HHfXFIXrtgv4P61pBa4Ypekxszkin21/XT3ryW1YiZX7Kvtp7t/LakVM7liB/fTJbVrJlfsktQyg12SGjMzWzEXHjD1QKmkls3Miv3CA6YeKJXUsplZsYMHTCXNhplZsUvSrGh6xe6+uqRZ1PSK3X11SbOo6RU7uK8uafY0vWKXpFlksEtSY5rYilnr7kceMJU0i5pYsa919yMPmEqaRU2s2MGDpJL0gSZW7JKkZQa7JDXGYJekxhjsktQYg12SGjO1Z8X4BV+StLqpXbH7BV+StLqpXbGD565L0mo6rdiTHE3ySpJDazy+PckLSRaTfGu0LUqSBtE32JPcB2ypqn3AriQ3r1L2EPC9qpoHrkgyP+I+JUkddVmxLwBP98bHgP2r1PwG+GiSDwEfAX4xku7W8NSJdzjx1tlxvoUkTa0uwb4V+OCrE88CO1ap+RFwA/Bl4Ke9uv8nycHeVs3i0tLSkO2e98HZMB4wlaQ/1SXY3wMu7423rfGcR4EvVtXXgJ8Bn19ZUFVHqmq+qubn5uaG7feP9t50NQ/u3XnRryNJrekS7CdZ3n7ZA5xapWY78LEkW4C9QI2kO0nSwLoE+7PAQ0keBz4DvJHk8IqafwaOAL8Drgb+faRdSpI663see1WdS7IAHAAeq6ozwOsran4M3DKWDiVJA+l0gVJVvcvymTGSpE1sar9SQJK0OoNdkhpjsEtSYwx2SWqMwS5JjTHYJakxBrskNcZgl6TGGOyS1BiDXZIaY7BLUmMMdklqjMEuSY0x2CWpMQa7JDVm6oL9qRPvcOKtP7lXtiSpZ+qC/bnXTgNw763Xb3AnkrQ5TV2wA+y96Woe3Ltzo9uQpE1pKoNdkrQ2g12SGmOwS1JjDHZJaozBLkmNMdglqTEGuyQ1xmCXpMYY7JLUGINdkhpjsEtSYwx2SWqMwS5JjekU7EmOJnklyaE+dU8kuWc0rUmShtE32JPcB2ypqn3AriQ3r1F3O/Dhqnp+xD1KkgbQZcW+ADzdGx8D9q8sSHIp8G3gVJJ7V3uRJAeTLCZZXFpaGrJdSVI/XYJ9K3C6Nz4L7Fil5nPAm8BjwG1JHl5ZUFVHqmq+qubn5uaG7VeS1EeXYH8PuLw33rbGcz4OHKmqM8B3gTtG054kaVBdgv0ky9sve4BTq9T8HNjVG88Db190Z5KkoVzSoeZZ4KUk1wF3Aw8kOVxVF54hcxT4tyQPAJcCnx59q5KkLvoGe1WdS7IAHAAe6223vL6i5vfAX4+lQ0nSQLqs2Kmqd1k+M0aStIl55akkNcZgl6TGGOyS1BiDXZIaY7BLUmMMdklqjMEuSY0x2CWpMQa7JDXGYJekxhjsktQYg12SGmOwS1JjDHZJaozBLkmNMdglqTEGuyQ1xmCXpMYY7JLUmKkK9qdOvMOJt85udBuStKlNVbA/99ppAO699foN7kSSNq+pCnaAvTddzYN7d250G5K0aU1dsEuS1mewS1JjDHZJaozBLkmNMdglqTEGuyQ1xmCXpMYY7JLUmE7BnuRokleSHOpTtyPJT0bTmiRpGH2DPcl9wJaq2gfsSnLzOuVfBy4fVXOSpMF1WbEvAE/3xseA/asVJbkT+ANwZiSdSZKG0iXYtwKne+OzwI6VBUkuA74KPLLWiyQ5mGQxyeLS0tIwvUqSOugS7O+xvL2ybY3nPAI8UVW/XetFqupIVc1X1fzc3NzgnUqSOukS7CdZ3n7ZA5xapeYu4EtJjgO3JvnOSLqTJA3skg41zwIvJbkOuBt4IMnhqvrjGTJV9ZcfjJMcr6q/HX2rkqQu+gZ7VZ1LsgAcAB6rqjPA6+vUL4ysO0nSwLqs2Kmqd1k+M0aStIl55akkNcZgl6TGGOyS1BiDXZIaY7BLUmMMdklqjMEuSY0x2CWpMQa7JDXGYJekxhjsktQYg12SGmOwS1JjDHZJaozBLkmNMdglqTEGuyQ1xmCXpMYY7JLUGINdkhpjsEtSYwx2SWqMwS5JjTHYJakxBrskNcZgl6TGGOyS1BiDXZIaY7BLUmMMdklqjMEuSY3pFOxJjiZ5JcmhNR6/KskPkhxL8v0kl422TUlSV32DPcl9wJaq2gfsSnLzKmWfBR6vqk8CZ4BPjbZNSVJXl3SoWQCe7o2PAfuB/7mwoKqeuOCPc8CvVr5IkoPAQYCdO3cO0aokqYsuWzFbgdO98Vlgx1qFSfYB26vq1ZWPVdWRqpqvqvm5ubmhmpUk9ddlxf4ecHlvvI01fhgkuRr4BvBXo2lNkjSMLiv2k5zffgHYA5xaWdA7WPoM8JWqentk3UmSBtYl2J8FHkryOPAZ4I0kh1fU/A3wCeAfkxxPcv+I+5QkddR3K6aqziVZAA4Aj1XVGeD1FTXfBL45lg4lSQPpssdOVb3L8pkxkqRNzCtPJakxBrskNcZgl6TGGOyS1BiDXZIaY7BLUmMMdklqjMEuSY0x2CWpMQa7JDXGYJekxhjsktQYg12SGmOwS1JjDHZJaozBLkmNMdglqTEGuyQ1xmCXpMYY7JLUGINdkhpjsEtSYwx2SWqMwS5JjTHYJakxBrskNcZgl6TGGOyS1BiDXZIaY7BLUmM6BXuSo0leSXLoYmokSePXN9iT3Adsqap9wK4kNw9TI0majC4r9gXg6d74GLB/yBpJ0gRc0qFmK3C6Nz4LfGKYmiQHgYMAO3fuHLhRgN3XXTnU8yRplnQJ9veAy3vjbay+yu9bU1VHgCMA8/PzNXCnwKP33DLM0yRppnTZijnJ8tbKHuDUkDWSpAnosmJ/FngpyXXA3cADSQ5X1aF1av589K1Kkrrou2KvqnOcPzj6KnBHVb2+ItRXq/nd6FuVJHXRZcVOVb3L8lkvQ9dIksbPK08lqTEGuyQ1xmCXpMYY7JLUmFQNda3Qxb1psgS8PeTTrwF+PcJ2poFzng3OeTZczJxvqKq5fkUbEuwXI8liVc1vdB+T5Jxng3OeDZOYs1sxktQYg12SGjONwX5koxvYAM55Njjn2TD2OU/dHrskaX3TuGKXJK3DYJekxmzaYJ/FG2j3m0+Sq5L8IMmxJN9Pctmkexy1rp9hkh1JfjKpvsZpgDk/keSeSfU1Th3+bm9P8kKSxSTfmnR/49D7O/vSOo9fmuT5JC8n+cIo33tTBvss3kC743w+CzxeVZ8EzgCfmmSPozbgZ/h1lu/SNbW6zjnJ7cCHq+r5iTY4Bh3n/BDwvd753Vckmepz25NsB57k/G1D1/IwcLKq/gL4dJIrRvX+mzLYmc0baC/QZz5V9URV/bD3xzngV5NpbWwW6PAZJrkT+APnf5hNuwX6zDnJpcC3gVNJ7p1ca2OzQP/P+TfAR5N8CPgI8IvJtDY27wP3A+fWqVlg+f/Li8DIfpht1mBfeXPsHUPWTJPO80myD9heVa9OorEx6jvn3nbTV4FHJtjXOHX5nD8HvAk8BtyW5OEJ9TYuXeb8I+AG4MvAT3t1U6uqznW44dDYMmyzBvtIbqA9ZTrNJ8nVwDeAke7JbZAuc34EeKKqfjuxrsary5w/DhypqjPAd4E7JtTbuHSZ86PAF6vqa8DPgM9PqLeNNLYM26xhOIs30O47n97q9RngK1U17JeobSZdPsO7gC8lOQ7cmuQ7k2ltbLrM+efArt54nuG/MG+z6DLn7cDHkmwB9gKzcIHN+DKsqjbdL+BK4HXgcc7/s2wPcLhPzVUb3fcE5vz3wLvA8d6v+ze673HPeUX98Y3ueUKf8xWc/wH+IvAKcP1G9z2BOd8GvMH5VewPgW0b3feI5n689/udwD+seOyG3pz/Bfhvzh9gHsn7btorT3tHlQ8AL9b5f5IOVTNNWptPF87ZOc+yJNdxftX+n9V/T777627WYJckDWez7rFLkoZksEtSYwx2SWqMwS5JjTHYJakx/we2ICDvsAdGxgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(fprs, tprs)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9830452674897119"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "roc_auc_score(y_test, decision_scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 多分类问题中的混淆矩阵"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 准备数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = digits.data\n",
    "y = digits.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=666)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n",
       "       3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n",
       "       3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n",
       "       3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n",
       "       3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n",
       "       3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n",
       "       3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n",
       "       3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,\n",
       "       3, 3, 3, 3, 3, 3, 3])"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[y==3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9555555555555556"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_reg = LogisticRegression(multi_class='ovr', solver='liblinear')\n",
    "log_reg.fit(X_train, y_train)\n",
    "log_reg.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_predict = log_reg.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## sklearn中混淆矩阵，精准率和召回率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9555555555555556"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import precision_score\n",
    "\n",
    "precision_score(y_test, y_predict, average=\"micro\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[45,  0,  0,  0,  0,  1,  0,  0,  0,  0],\n",
       "       [ 0, 37,  0,  0,  0,  0,  0,  0,  3,  0],\n",
       "       [ 0,  0, 49,  1,  0,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0, 49,  0,  1,  0,  0,  3,  0],\n",
       "       [ 0,  1,  0,  0, 47,  0,  0,  0,  0,  0],\n",
       "       [ 0,  0,  0,  1,  0, 36,  0,  0,  1,  0],\n",
       "       [ 0,  0,  0,  0,  0,  1, 38,  0,  0,  0],\n",
       "       [ 0,  0,  0,  0,  0,  0,  0, 42,  0,  1],\n",
       "       [ 0,  2,  0,  0,  0,  0,  0,  0, 46,  0],\n",
       "       [ 0,  1,  0,  1,  1,  1,  0,  0,  0, 41]], dtype=int64)"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test, y_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP0AAAEACAYAAAB4TnCPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAACalJREFUeJzt3U+IXeUZgPHnbWYCmtpJTFOlMAhCVlWqZdAqtUZJxS5E2hosiAHbMliKriOtSF20IOKmkOJY7Ub7z0ILggWbQqKCUlIkm2jRhUGEYDs2aY2LjuTtIlcNJs49k97vnhzf57e6kzl+9zXj47n/zjeRmUiq41N9DyBpuoxeKsbopWKMXirG6KVijF4qxug7iIi5iPhTRDwTEX+IiPV9z9RFRFwQES/1PcdaRMTuiLip7zm6iIhNEfF0ROyPiIf7nqerXqKPiEcj4oWI+FEf938GbgMeyswbgMPAjT3P09WDwDl9D9FVRFwDXJiZT/U9S0e3A09k5gJwXkQs9D1QF1OPPiK+CazLzKuAiyNi67RnWKvM3J2Zfx59uQV4q895uoiI64FjnPif1FkvImaBR4DXI+LmvufpaBm4JCI2AvPAGz3P00kfZ/ptwO9Gt58BvtLDDGckIq4CNmXmi33PsprR0497gV19z7IGO4GDwAPAFRFxV8/zdPE8cBFwN/Ay8Ha/43TTR/QbgDdHt98GLuhhhjWLiPOBnwHf6XuWDnYBuzPzSN+DrMHlwFJmHgYeB67reZ4u7gPuzMz7gVeAO3qep5M+on+HD59nfrqnGdZkdOZ8ErgnMw/1PU8H24EfRMRe4LKI+EXP83TxGnDx6PYCMIS/503ApRGxDrgSGMSFLDHtC24iYifwucx8MCJ+DPw9M3811SHWKCK+D/wEODD6o59n5m97HKmziNibmdv6nmOciDgPeIwTj/xmgVsy883V/6l+RcQVwC858RD/BeAbmflOv1ON10f0nwGeA/4CfB34cmYeneoQUmFTjx5OvL8JfA14dvQcTtKU9BK9pP6c9S+iSZoso5eK6S36iFjs677PlDO3N7R5YXgz93mmH9Rf1Igztze0eWFgM/vwXipmoq/eb968Oefn5zsdu7y8zObNmzsde+DAgfEH6f8WEU3W9R2i6cjMTj/AmUne6fz8PHv27JnkkgBs2bJl4mvqVDMzE/3P4QMrKytN1tWZ8eG9VIzRS8UYvVSM0UvFGL1UjNFLxXSKfoC710r6GGOjH+LutZI+Xpcz/TYGunutpFN1iX7V3WsjYnH0Gz72Ly8vT3o+SRPWJfpVd6/NzKXMXMjMha6fpZfUny7R/40PH9J/EXi92TSSmutyhcUfgeci4vOMdq9tO5Kklsae6TPz35x4Me9F4Dq3q5aGrdO1lJn5Lz58BV/SgPmJPKkYo5eKMXqpGKOXipnoxpgR0WQHxH379rVYFoBrr7222doarrm5uWZrHz3a5g2wrhtjeqaXijF6qRijl4oxeqkYo5eKMXqpGKOXijF6qRijl4oxeqkYo5eKMXqpGKOXijF6qRijl4oxeqkYo5eKMXqpGKOXijF6qRijl4oxeqmYQWyB3dIk//1Ptn79+ibrAqysrDRbW8PlFtiSTsvopWKMXirG6KVijF4qxuilYoxeKsbopWJmxh0QEXPAb4B1wDHg1sz8b+vBJLXR5Ux/G/BQZt4AHAZubDuSpJbGnukzc/dJX24B3mo3jqTWxkb/voi4CtiUmS9+5M8XgcVJDyapjU4X3ETE+cAzwLcy89Aqx3nBzYgX3GjaJnbBTUSsB54E7lkteEnD0OWFvO8CXwJ+GBF7I+LWxjNJasjr6X14r08Ir6eXdFpGLxVj9FIxRi8VY/RSMZ0/kfdJFdHpBc81a/WuALSbeXZ2tsm6Q3y3YW5urtnaR48ebbZ2F57ppWKMXirG6KVijF4qxuilYoxeKsbopWKMXirG6KVijF4qxuilYoxeKsbopWKMXirG6KVijF4qxuilYoxeKsbopWKMXirG6KVijF4qZhC/wLLV1swwzO2Zjx071mTdDRs2NFlX0+EvsJR0WkYvFWP0UjFGLxVj9FIxRi8VY/RSMZ2ij4gLIuKl1sNIaq/rmf5B4JyWg0iajrHRR8T1wDHgcPtxJLW2avQRsR64F9g1nXEktTYz5vu7gN2ZeSTi9B/rjYhFYHHSg0lqY9ULbiLiWeD46MvLgN9n5vdWOd4LbqbAC250Ol0vuFn1TJ+ZX33/dkTsXS14ScPQ+X36zNzWcA5JU+KHc6RijF4qxuilYoxeKsbopWKMXipmELvhDtEQP1C0Z8+eJutu3769ybotDfHn5264kk7L6KVijF4qxuilYoxeKsbopWKMXirG6KVijF4qxuilYoxeKsbopWKMXirG6KVijF4qxuilYoxeKsbopWKMXirG6KVijF4qxt1w9YFWO8Du27evyboAV199dbO1h8bdcCWdltFLxRi9VIzRS8UYvVSM0UvFGL1UjNFLxXSOPiJ2R8RNLYeR1F6n6CPiGuDCzHyq8TySGhsbfUTMAo8Ar0fEze1HktRSlzP9TuAg8ABwRUTcdfI3I2IxIvZHxP4WA0qarC7RXw4sZeZh4HHgupO/mZlLmbmQmQstBpQ0WV2ifw24eHR7ATjUbhxJrc10OOZR4LGI+DYwC9zSdiRJLY2NPjP/A+yYwiySpsAP50jFGL1UjNFLxRi9VIzRS8UYvVSMW2Br0F599dUm627durXJutBmq/H33nuP48ePuwW2pFMZvVSM0UvFGL1UjNFLxRi9VIzRS8UYvVSM0UvFGL1UjNFLxRi9VIzRS8UYvVSM0UvFGL1UjNFLxRi9VIzRS8UYvVSM0UvFDGI33HPPPbfFsgC8++67zdbWcB05cqTZ2hs3bmyybma6G66kUxm9VIzRS8UYvVSM0UvFGL1UjNFLxawafURsioinI2J/RDw8raEktTPuTH878ERmLgDnRcTCFGaS1NC46JeBSyJiIzAPvNF+JEktjYv+eeAi4G7gZeDt5hNJampc9PcBd2bm/cArwB0fPSAiFkfP+fe3GFDSZI2LfhNwaUSsA64ETrmgJjOXMnNh9Lxf0lluXPQ/BZaAo8D5wK+bTySpqZnVvpmZfwW+MKVZJE2BH86RijF6qRijl4oxeqkYo5eKMXqpGKOXiln1ffqzxcrKSt8jrNns7GzfI5w1hvjza7VNNcDBgwcnvuaOHTs6H+uZXirG6KVijF4qxuilYoxeKsbopWKMXirG6KVijF4qxuilYoxeKsbopWKMXirG6KVijF4qxuilYoxeKsbopWKMXirG6KVijF4qJjJP+ZXzZ75YxD+AQx0P/yzwz4nd+XQ4c3tDmxfOjpkvyswtXQ6caPRrERH7M3Ohlzs/Q87c3tDmheHN7MN7qRijl4rpM/qlHu/7TDlze0ObFwY2c2/P6SX1w4f3UjFGLxVj9FIxRi8VY/RSMf8DGFAXNXyYkxcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cfm = confusion_matrix(y_test, y_predict)\n",
    "plt.matshow(cfm, cmap=plt.cm.gray)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.02631579, 0.        , 0.        , 0.        , 0.        ],\n",
       "       [0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.0625    , 0.        ],\n",
       "       [0.        , 0.        , 0.        , 0.01886792, 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ],\n",
       "       [0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.02631579, 0.        , 0.        , 0.0625    , 0.        ],\n",
       "       [0.        , 0.025     , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ],\n",
       "       [0.        , 0.        , 0.        , 0.01886792, 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.02083333, 0.        ],\n",
       "       [0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.02631579, 0.        , 0.        , 0.        , 0.        ],\n",
       "       [0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.02222222],\n",
       "       [0.        , 0.05      , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ],\n",
       "       [0.        , 0.025     , 0.        , 0.01886792, 0.02083333,\n",
       "        0.02631579, 0.        , 0.        , 0.        , 0.        ]])"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "row_sums = np.sum(cfm, axis=1)\n",
    "err_matrix = cfm / row_sums\n",
    "np.fill_diagonal(err_matrix, 0)\n",
    "\n",
    "err_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP0AAAEACAYAAAB4TnCPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAACUJJREFUeJzt3U+I3OUZwPHv0yQSaa1GmigFEYSeqlTLoA3UkogVe1ikrdFCULAti2XRc6QVqYcWRLzIJjj+6UX7L4cWFgxYCqEKStkivfgHPCSIEEyTaquXXp4edtpoEnd+u8w7v/x4vp/TZGfy7kPgm3dm9rfvRGYiqY7P9T2ApPkyeqkYo5eKMXqpGKOXijF6qRij7yAiLo2IIxHxUkT8ISIu6numLiLiioh4ve85NiIiDkbEQt9zdBEROyLixYhYjYin+p6nq16ij4hnI+LViPhZH99/E/YDT2TmbcAJ4Pae5+nqceDivofoKiJuBq7MzJW+Z+noHuCFzBwBl0TEqO+Buph79BHxPWBLZu4GromIr8x7ho3KzIOZ+afJH3cC7/c5TxcRcQvwMWv/SV3wImIb8DRwLCLu6Huejk4B10bEZcBVwLs9z9NJHzv9HuD3k9svAd/sYYZNiYjdwI7MfK3vWdYzefnxMHCg71k24F7gDeAx4MaIeKDnebp4BbgaeBB4Ezjd7zjd9BH954H3JrdPA1f0MMOGRcTlwJPAD/uepYMDwMHM/KDvQTbgBmCcmSeA54G9Pc/TxSPA/Zn5KPAWcF/P83TSR/QfceZ15hd6mmFDJjvnYeChzDze9zwd3AosRcRR4PqIeKbnebp4B7hmcnsEDOHfeQdwXURsAW4CBvGLLDHvX7iJiHuBXZn5eET8HHg7M3891yE2KCJ+AvwC+PvkS4cy83c9jtRZRBzNzD19zzFNRFwCPMfaM79twJ2Z+d76f6tfEXEj8CvWnuK/Cnw3Mz/qd6rp+oj+i8DLwJ+B7wDfyMwP5zqEVNjco4e1n28C3wb+MnkNJ2lOeoleUn8u+DfRJM2W0UvF9BZ9RCz29b03y5nbG9q8MLyZ+9zpB/UPNeHM7Q1tXhjYzD69l4qZ6bv3EeGPAgZs586dTdY9efJkk3X1aZkZXR63tfUgGo677rqrybrLy8tN1tXm+PReKsbopWKMXirG6KVijF4qxuilYjpFP8DTayV9hqnRD/H0WkmfrctOv4eBnl4r6Vxdol/39NqIWJx8wsfqrIeTNHtdLsNd9/TazBwDY/Dae2kIuuz0f+PMU/qvAceaTSOpuS47/R+BlyPiy0xOr207kqSWpu70mfkv1t7Mew3Y63HV0rB1+tXazPwnZ97BlzRgXpEnFWP0UjFGLxVj9FIxHowpnUfLj3uL6HR+5YZ1PRjTnV4qxuilYoxeKsbopWKMXirG6KVijF4qxuilYoxeKsbopWKMXirG6KVijF4qxuilYoxeKsbopWKMXirG6KVijF4qxuilYoxeKsbopWI6fZadNm5hYaHZ2isrK83W1ppWx1RfCNzppWKMXirG6KVijF4qxuilYoxeKsbopWKMXipm6sU5EXEp8FtgC/AxcHdm/qf1YJLa6LLT7weeyMzbgBPA7W1HktTS1J0+Mw9+4o87gffbjSOptc7X3kfEbmBHZr521tcXgcVZDyapjU7RR8TlwJPA98++LzPHwHjyuJzpdJJmbupr+oi4CDgMPJSZx9uPJKmlLm/k/Qj4OvDTiDgaEXc3nklSQ13eyDsEHJrDLJLmwItzpGKMXirG6KVijF4qxuilYiJzdtfTeHHOsC0tLTVZd3l5ucm6Lc2yi7O1Omk3Mzst7E4vFWP0UjFGLxVj9FIxRi8VY/RSMUYvFWP0UjFGLxVj9FIxRi8VY/RSMUYvFWP0UjFGLxVj9FIxRi8VY/RSMUYvFWP0UjFGLxVj9FIxgzgCe3FxscWyAIzH42ZrS/PkEdiSzsvopWKMXirG6KVijF4qxuilYoxeKqZT9BFxRUS83noYSe113ekfBy5uOYik+ZgafUTcAnwMnGg/jqTW1o0+Ii4CHgYOzGccSa1tnXL/AeBgZn4Qcf7LeiNiEWh3cbykmZr29P5WYCkijgLXR8QzZz8gM8eZOcrMUYsBJc3Wujt9Zn7rf7cj4mhm/rj9SJJa6vxz+szc03AOSXPixTlSMUYvFWP0UjFGLxVj9FIxRi8VM4jTcIdoYWGh2dorKyvN1taaffv2NVv78OHDTdb1NFxJ52X0UjFGLxVj9FIxRi8VY/RSMUYvFWP0UjFGLxVj9FIxRi8VY/RSMUYvFWP0UjFGLxVj9FIxRi8VY/RSMUYvFWP0UjFGLxXjabj6v6WlpSbrLi8vN1lXn+ZpuJLOy+ilYoxeKsbopWKMXirG6KVijF4qxuilYjpHHxEHI6Ld5y9LmotO0UfEzcCVmekHo0sDNzX6iNgGPA0ci4g72o8kqaUuO/29wBvAY8CNEfHAJ++MiMWIWI2I1RYDSpqtLtHfAIwz8wTwPLD3k3dm5jgzR5k5ajGgpNnqEv07wDWT2yPgeLtxJLW2tcNjngWei4gfANuAO9uOJKmlqdFn5r+BfXOYRdIceHGOVIzRS8UYvVSM0UvFGL1UjNFLxXgEtjRn+/fvn/maR44c4dSpUx6BLelcRi8VY/RSMUYvFWP0UjFGLxVj9FIxRi8VY/RSMUYvFWP0UjFGLxVj9FIxRi8VY/RSMUYvFWP0UjFGLxVj9FIxRi8VY/RSMYM4DXd1dbXFsgCMRqNma0vzlJmehivpXEYvFWP0UjFGLxVj9FIxRi8VY/RSMetGHxE7IuLFiFiNiKfmNZSkdqbt9PcAL2TmCLgkIrySRRq4adGfAq6NiMuAq4B3248kqaVp0b8CXA08CLwJnG4+kaSmpkX/CHB/Zj4KvAXcd/YDImJx8pq/3QXykmZmWvQ7gOsiYgtwE3DOL9Rk5jgzR5PX/ZIucNOi/yUwBj4ELgd+03wiSU1tXe/OzPwr8NU5zSJpDrw4RyrG6KVijF4qxuilYoxeKsbopWKMXipm3Z/TXyjG43HfI2zYwsJCs7W3b9/eZN1du3Y1WXd5ebnJutocd3qpGKOXijF6qRijl4oxeqkYo5eKMXqpGKOXijF6qRijl4oxeqkYo5eKMXqpGKOXijF6qRijl4oxeqkYo5eKMXqpGKOXijF6qZjIPOcj5ze/WMRJ4HjHh38J+MfMvvl8OHN7Q5sXLoyZr87MnV0eONPoNyIiVjNz1Ms33yRnbm9o88LwZvbpvVSM0UvF9Bn98D62xpnnYWjzwsBm7u01vaR++PReKsbopWKMXirG6KVijF4q5r/kitErDXrJygAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.matshow(err_matrix, cmap=plt.cm.gray)\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "314.438px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
